Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
173c5d5ef4 | ||
|
|
c7c784161d | ||
|
|
0fdacb9839 | ||
|
|
b0e4a12c88 | ||
|
|
a76b0742ff | ||
|
|
21a499ba59 | ||
|
|
de9d29dc65 | ||
|
|
62ec68355c | ||
|
|
6a5e6510ca | ||
|
|
3f97d2d50d | ||
|
|
6112233649 | ||
|
|
8c2d7e5f97 | ||
|
|
21ebbffed4 | ||
|
|
31120ed580 | ||
|
|
cf304d8629 | ||
|
|
12d7852082 | ||
|
|
c9848f4fb8 | ||
|
|
c070bea782 | ||
|
|
7443938fef | ||
|
|
aee97bf8ad | ||
|
|
c816cafc00 | ||
|
|
2353350d79 | ||
|
|
d4b8b24701 | ||
|
|
de6657c846 | ||
|
|
f75e1a91d9 | ||
|
|
ebfec39937 | ||
|
|
24d05701d3 | ||
|
|
a95b3fa053 | ||
|
|
c2cd2e1eb1 | ||
|
|
aa62de2e78 | ||
|
|
66713c5a1e | ||
|
|
b771794f1c | ||
|
|
6c63a17cfa | ||
|
|
ed66a2e873 | ||
|
|
cc73b38671 | ||
|
|
0993c31ad4 | ||
|
|
82475691b2 | ||
|
|
89046e6613 | ||
|
|
dc89995d96 | ||
|
|
6c10288f0b | ||
|
|
789b0a9474 | ||
|
|
eb97a3d9a2 | ||
|
|
8b2e2300d8 | ||
|
|
2f2daea415 | ||
|
|
3b48f5962a | ||
|
|
2b1678f315 | ||
|
|
ab739d02dd | ||
|
|
a60020b51a | ||
|
|
a8d84d1730 | ||
|
|
9e8521afb5 | ||
|
|
8c60604eb7 | ||
|
|
13cd710071 | ||
|
|
0914a33fba | ||
|
|
b5417c28d0 | ||
|
|
5f6a41146c | ||
|
|
c71c513027 | ||
|
|
2e32f47e45 | ||
|
|
9492278d93 | ||
|
|
5ce2d07c7f | ||
|
|
373f339750 | ||
|
|
48f0662fde | ||
|
|
7913b56d66 | ||
|
|
ab1811d003 | ||
|
|
c79337579d | ||
|
|
3a84044ece | ||
|
|
c51357f8af | ||
|
|
2833a049aa | ||
|
|
6f7f07635e | ||
|
|
0f76b6e232 | ||
|
|
3e16c055be | ||
|
|
0361c44a03 | ||
|
|
d1debdcce8 | ||
|
|
be53f5167a | ||
|
|
2130f99612 | ||
|
|
ed5690cee4 | ||
|
|
70c1110c53 |
@@ -0,0 +1,3 @@
|
|||||||
|
module git.novox.be/novox/mesh-catalog/checks/mailstore
|
||||||
|
|
||||||
|
go 1.22
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
// Command mailstore refuses any manifest but modules/mesh-mail claiming the mail-store seat (novox/hq ADR 0378
|
||||||
|
// §3, issue 800 part A): the operator's mail is read, searched and sent through this one seat, and a second
|
||||||
|
// claimant would be a second path to the same mail that ADR 0378 never accounted for.
|
||||||
|
//
|
||||||
|
// go run . <catalogue root>
|
||||||
|
//
|
||||||
|
// Exits 1 on a finding.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MailStoreSeat is the seat only mesh-mail may claim.
|
||||||
|
const MailStoreSeat = "mail-store"
|
||||||
|
|
||||||
|
// MailStoreModule is the one module allowed to claim it.
|
||||||
|
const MailStoreModule = "mesh-mail"
|
||||||
|
|
||||||
|
// Finding is one manifest refused: a module other than mesh-mail claiming mail-store.
|
||||||
|
type Finding struct {
|
||||||
|
Module string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check reads every manifest under root/modules and answers which ones it refuses.
|
||||||
|
func Check(root string) ([]Finding, error) {
|
||||||
|
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sort.Strings(manifests)
|
||||||
|
var out []Finding
|
||||||
|
for _, path := range manifests {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Claims []struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
} `json:"claims"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: %w", path, err)
|
||||||
|
}
|
||||||
|
if m.Module == MailStoreModule {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if c.Name == MailStoreSeat {
|
||||||
|
out = append(out, Finding{Module: m.Module})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
root := "."
|
||||||
|
if len(os.Args) > 1 {
|
||||||
|
root = os.Args[1]
|
||||||
|
}
|
||||||
|
found, err := Check(root)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
for _, f := range found {
|
||||||
|
fmt.Printf("%s claims the mail-store seat: only %s may (novox/hq ADR 0378 §3)\n", f.Module, MailStoreModule)
|
||||||
|
}
|
||||||
|
if len(found) > 0 {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
fmt.Println("mail-store: claimed only by mesh-mail")
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func manifest(t *testing.T, root, module, claims string) {
|
||||||
|
t.Helper()
|
||||||
|
dir := filepath.Join(root, "modules", module)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body := `{"module":"` + module + `","claims":[` + claims + `]}`
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnotherModuleClaimingMailStoreIsRefused(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
manifest(t, root, "mesh-mail", `{"name":"mail-store","scope":"mesh"}`)
|
||||||
|
manifest(t, root, "betterbird", `{"name":"mail-store","scope":"mesh"}`)
|
||||||
|
manifest(t, root, "dunst", `{"name":"node-notifier","scope":"node"}`)
|
||||||
|
found, err := Check(root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(found) != 1 || found[0].Module != "betterbird" {
|
||||||
|
t.Fatalf("refused %+v", found)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCatalogueHasNone(t *testing.T) {
|
||||||
|
found, err := Check("../..")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(found) != 0 {
|
||||||
|
t.Fatalf("%+v", found)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,9 +23,51 @@
|
|||||||
#
|
#
|
||||||
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
|
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
|
||||||
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
|
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
|
||||||
|
#
|
||||||
|
# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is
|
||||||
|
# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage
|
||||||
|
# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this
|
||||||
|
# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last
|
||||||
|
# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a
|
||||||
|
# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm
|
||||||
|
# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor
|
||||||
|
# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is
|
||||||
|
# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner).
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
|
work=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$work"' EXIT
|
||||||
|
|
||||||
deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
|
deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
|
||||||
|
if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then
|
||||||
|
origin=$(git remote get-url origin 2>/dev/null || true)
|
||||||
|
forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}"
|
||||||
|
owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}"
|
||||||
|
if [ -z "$forge" ] || [ -z "$owner" ]; then
|
||||||
|
echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)"
|
||||||
|
tc="$work/toolchain"
|
||||||
|
mkdir -p "$tc"
|
||||||
|
printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc"
|
||||||
|
# Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails.
|
||||||
|
if ! { node -e '
|
||||||
|
const [base, dir] = process.argv.slice(1);
|
||||||
|
Promise.all(["package.json", "package-lock.json"].map(async (f) => {
|
||||||
|
const r = await fetch(base + f);
|
||||||
|
if (!r.ok) throw new Error(base + f + ": " + r.status);
|
||||||
|
require("fs").writeFileSync(dir + "/" + f, await r.text());
|
||||||
|
})).catch((e) => { console.error(e.message); process.exit(1); });' \
|
||||||
|
"$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" &&
|
||||||
|
(cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" &&
|
||||||
|
npm ci --ignore-scripts --no-audit --no-fund --loglevel=error &&
|
||||||
|
npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then
|
||||||
|
echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
deps="$tc/node_modules"
|
||||||
|
fi
|
||||||
tsc="$deps/typescript/bin/tsc"
|
tsc="$deps/typescript/bin/tsc"
|
||||||
if [ ! -f "$tsc" ]; then
|
if [ ! -f "$tsc" ]; then
|
||||||
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
|
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
|
||||||
@@ -39,8 +81,6 @@ touched=$({
|
|||||||
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
|
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
|
||||||
} | sed '/^$/d' | sort -u)
|
} | sed '/^$/d' | sort -u)
|
||||||
|
|
||||||
work=$(mktemp -d)
|
|
||||||
trap 'rm -rf "$work"' EXIT
|
|
||||||
failed=""
|
failed=""
|
||||||
for m in $touched; do
|
for m in $touched; do
|
||||||
d="modules/$m"
|
d="modules/$m"
|
||||||
|
|||||||
@@ -33,11 +33,14 @@
|
|||||||
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
|
||||||
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
|
||||||
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
# the nested ones the controller cannot protect alone yet, and only shrinks.
|
||||||
|
# 7. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
(cd checks/words && go test -count=1 . && go run . ../..)
|
(cd checks/words && go test -count=1 . && go run . ../..)
|
||||||
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
|
||||||
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
(cd checks/agenthome && go test -count=1 . && go run . ../..)
|
||||||
|
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
|
||||||
|
(cd checks/mailstore && go test -count=1 . && go run . ../..)
|
||||||
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
|
||||||
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
|
||||||
secrets_base=""
|
secrets_base=""
|
||||||
|
|||||||
@@ -7,9 +7,15 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
|
|||||||
- Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires
|
- Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires
|
||||||
`x11-display` on its own machine.
|
`x11-display` on its own machine.
|
||||||
- **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it.
|
- **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it.
|
||||||
- Starts `clipmenud` **once per session**, from the session's start (the `xinitrc` slot `normal`).
|
- Declares `clipmenud.service` as a **user unit of the operator's account**, enabled and running
|
||||||
It is not a user unit as well. Its packaged unit needs user-scoped units (mesh-host #72, not
|
(`scope: user`, ADR 0177; issue 743), so `$mod+.` shows the history again. It is the only starter: the
|
||||||
merged), and the session start alone is one starter.
|
session's `xinitrc` line is gone. A drop-in (`~/.config/systemd/user/clipmenud.service.d/mesh.conf`)
|
||||||
|
replaces `ExecStart` with the module's launcher, which finds the X display from its socket (the account's
|
||||||
|
service manager has no `DISPLAY`) and puts the text-only `xsel` first on the `PATH`; until the display is up
|
||||||
|
it waits (a quick exit would run into the start limit, which the drop-in also switches off, with `RestartSec=2s`). `DISPLAY` and `XAUTHORITY` come from the user manager when the session's xinitrc has handed them over (module xorg); with several X servers and no `DISPLAY` yet, it waits for them. The store is in the account's runtime directory (mode 0700),
|
||||||
|
so no other account reads it.
|
||||||
|
**After delivery the operator logs in again**: the running session keeps the `clipmenud` its xinitrc
|
||||||
|
started, and the new unit waits on clipmenud's session lock until that one ends.
|
||||||
- Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the
|
- Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the
|
||||||
history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu.
|
history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu.
|
||||||
- Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools
|
- Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools
|
||||||
@@ -23,11 +29,11 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
|
|||||||
|
|
||||||
| tool | does |
|
| tool | does |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `node-clipboard.history` | the history, newest first, each entry once with its id, first line, time, size and text (cut) |
|
| `node-clipboard.history` | the history, newest first: count, time and length of each entry, and whether the daemon collects. **Never a text, a first line or an id** (issue 750); the text is read through clipmenu on the operator's own screen |
|
||||||
| `node-clipboard.copy` | put text on the clipboard; it enters the history like any copy |
|
| `node-clipboard.copy` | put text on the clipboard, then read it back; an error with the reason (no `DISPLAY`, `xsel` exited, other contents) when it does not hold the text. It enters the history like any copy |
|
||||||
| `clipmenu_paste` | what the clipboard holds now |
|
| `clipmenu_paste` | how many bytes of text the clipboard holds now, never the text |
|
||||||
| `clipmenu_clear` | forget the history; the daemon's locks stay |
|
| `clipmenu_clear` | forget the history; the daemon's locks stay |
|
||||||
| `clipmenu_delete` | forget one entry, by id or first line |
|
| `clipmenu_delete` | forget one entry, by its place in the history (1 is the newest) and the time the history gave it, refused if the history changed since; not by text or id |
|
||||||
|
|
||||||
The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's
|
The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's
|
||||||
own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an
|
own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
mostCopy = 1 << 20
|
mostCopy = 1 << 20
|
||||||
mostPaste = 64 << 10
|
|
||||||
// majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/.
|
// majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/.
|
||||||
majorVersion = 6
|
majorVersion = 6
|
||||||
)
|
)
|
||||||
@@ -71,15 +70,15 @@ func cksum(data []byte) string {
|
|||||||
|
|
||||||
func entryID(line string) string { return cksum([]byte(line + "\n")) }
|
func entryID(line string) string { return cksum([]byte(line + "\n")) }
|
||||||
|
|
||||||
// Entry is one clip in the history.
|
// Entry is one clip in the history as a caller sees it: when and how long, never what. The text of a
|
||||||
|
// clip, its first line and its id (a checksum of that line) stay in clipmenu's store, readable on the
|
||||||
|
// operator's own screen through clipmenu (issue 750).
|
||||||
type Entry struct {
|
type Entry struct {
|
||||||
ID string `json:"id"`
|
At string `json:"at"`
|
||||||
Line string `json:"line"`
|
Bytes int `json:"bytes"`
|
||||||
At string `json:"at"`
|
id string
|
||||||
Bytes int `json:"bytes"`
|
line string
|
||||||
Text string `json:"text,omitempty"`
|
at int64
|
||||||
Truncated bool `json:"truncated,omitempty"`
|
|
||||||
at int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// HistoryResult is what node-clipboard.history answers.
|
// HistoryResult is what node-clipboard.history answers.
|
||||||
@@ -119,14 +118,15 @@ func readStore(dir string) ([]Entry, error) {
|
|||||||
}
|
}
|
||||||
out := make([]Entry, 0, len(latest))
|
out := make([]Entry, 0, len(latest))
|
||||||
for line, ns := range latest {
|
for line, ns := range latest {
|
||||||
out = append(out, Entry{ID: entryID(line), Line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)})
|
out = append(out, Entry{id: entryID(line), line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)})
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at })
|
sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at })
|
||||||
return out, scan.Err()
|
return out, scan.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// History is the clipboard's history, newest first.
|
// History is the clipboard's history, newest first: how many entries, when each was copied and how
|
||||||
func History(limit, maxBytes int) (HistoryResult, error) {
|
// long each is, and whether the daemon is collecting. It never returns a copied text.
|
||||||
|
func History(limit int) (HistoryResult, error) {
|
||||||
dir, err := storeDir(findEnvironment())
|
dir, err := storeDir(findEnvironment())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return HistoryResult{}, err
|
return HistoryResult{}, err
|
||||||
@@ -140,15 +140,8 @@ func History(limit, maxBytes int) (HistoryResult, error) {
|
|||||||
if i == limit {
|
if i == limit {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if info, err := os.Stat(filepath.Join(dir, e.ID)); err == nil {
|
if info, err := os.Stat(filepath.Join(dir, e.id)); err == nil {
|
||||||
e.Bytes = int(info.Size())
|
e.Bytes = int(info.Size())
|
||||||
if maxBytes > 0 {
|
|
||||||
raw, _ := os.ReadFile(filepath.Join(dir, e.ID))
|
|
||||||
if len(raw) > maxBytes {
|
|
||||||
raw, e.Truncated = raw[:maxBytes], true
|
|
||||||
}
|
|
||||||
e.Text = string(raw)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
out.Entries = append(out.Entries, e)
|
out.Entries = append(out.Entries, e)
|
||||||
}
|
}
|
||||||
@@ -159,6 +152,40 @@ func History(limit, maxBytes int) (HistoryResult, error) {
|
|||||||
type CopyResult struct {
|
type CopyResult struct {
|
||||||
Bytes int `json:"bytes"`
|
Bytes int `json:"bytes"`
|
||||||
Unit string `json:"unit"`
|
Unit string `json:"unit"`
|
||||||
|
// Verified says the clipboard was read back and held exactly the text.
|
||||||
|
Verified bool `json:"verified"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readBackWait is the pause between reads of the clipboard while the owner starts; read-back tries 15 times.
|
||||||
|
var readBackWait = 200 * time.Millisecond
|
||||||
|
|
||||||
|
// readBack waits for the clipboard to hold text and answers why when it does not.
|
||||||
|
func readBack(s Session, unit, text string) error {
|
||||||
|
var last string
|
||||||
|
for i := 0; i < 15; i++ {
|
||||||
|
time.Sleep(readBackWait)
|
||||||
|
r, err := s.run(3*time.Second, "", "xsel", "--output", "--clipboard")
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
last = err.Error()
|
||||||
|
case r.Code != 0:
|
||||||
|
last = "xsel: " + strings.TrimSpace(r.Stderr)
|
||||||
|
case r.Stdout == text:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
last = fmt.Sprintf("it holds %d bytes of something else", len(r.Stdout))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
why := ""
|
||||||
|
if s.Display == "" {
|
||||||
|
why = "; the session has no DISPLAY, so the owner could not reach the screen"
|
||||||
|
}
|
||||||
|
if j, err := s.run(5*time.Second, "", "journalctl", "--user", "-u", unit+".service", "-n", "5", "--no-pager", "-o", "cat"); err == nil {
|
||||||
|
if tail := strings.TrimSpace(j.Stdout); tail != "" {
|
||||||
|
why += "; the owner said: " + tail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the clipboard does not hold the text (%d bytes): %s%s", len(text), last, why)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the
|
// Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the
|
||||||
@@ -194,17 +221,20 @@ func Copy(text string) (CopyResult, error) {
|
|||||||
os.Remove(f.Name())
|
os.Remove(f.Name())
|
||||||
return CopyResult{}, err
|
return CopyResult{}, err
|
||||||
}
|
}
|
||||||
return CopyResult{Bytes: len(text), Unit: unit + ".service"}, nil
|
if err := readBack(s, unit, text); err != nil {
|
||||||
|
return CopyResult{}, err
|
||||||
|
}
|
||||||
|
return CopyResult{Bytes: len(text), Unit: unit + ".service", Verified: true}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PasteResult is what clipmenu_paste answers.
|
// PasteResult is what clipmenu_paste answers: how much text the clipboard holds, never the text. The
|
||||||
|
// clipboard is the newest history entry, and a copied secret must not reach a caller (issue 750).
|
||||||
type PasteResult struct {
|
type PasteResult struct {
|
||||||
Text string `json:"text"`
|
Bytes int `json:"bytes"`
|
||||||
Bytes int `json:"bytes"`
|
Text bool `json:"holds_text"`
|
||||||
Truncated bool `json:"truncated,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Paste reads the clipboard now.
|
// Paste reads the clipboard now and says how long it is.
|
||||||
func Paste() (PasteResult, error) {
|
func Paste() (PasteResult, error) {
|
||||||
s, err := findSession()
|
s, err := findSession()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -217,11 +247,7 @@ func Paste() (PasteResult, error) {
|
|||||||
if r.Code != 0 {
|
if r.Code != 0 {
|
||||||
return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr))
|
return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr))
|
||||||
}
|
}
|
||||||
out := PasteResult{Text: r.Stdout, Bytes: len(r.Stdout), Truncated: r.Truncated}
|
return PasteResult{Bytes: len(r.Stdout), Text: len(r.Stdout) > 0}, nil
|
||||||
if len(out.Text) > mostPaste {
|
|
||||||
out.Text, out.Truncated = out.Text[:mostPaste], true
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ChangeResult is what clear and delete answer.
|
// ChangeResult is what clear and delete answer.
|
||||||
@@ -298,10 +324,12 @@ func Clear() (ChangeResult, error) {
|
|||||||
return out, err
|
return out, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete forgets one entry, by id or by its first line.
|
// Delete forgets one entry, by its place in the history as node-clipboard.history lists it (1 is the
|
||||||
func Delete(id, line string) (ChangeResult, error) {
|
// newest) together with the time it reports, so a copy arriving since the listing cannot shift the
|
||||||
if (id == "") == (line == "") {
|
// place onto another entry. Not by its text or id: either would let a caller confirm a guess of a copied line.
|
||||||
return ChangeResult{}, errors.New("give the entry's id or its line, one of the two")
|
func Delete(place int, at string) (ChangeResult, error) {
|
||||||
|
if place < 1 {
|
||||||
|
return ChangeResult{}, errors.New("give the entry's place in the history, 1 for the newest")
|
||||||
}
|
}
|
||||||
dir, err := storeOf()
|
dir, err := storeOf()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -309,6 +337,17 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
}
|
}
|
||||||
var out ChangeResult
|
var out ChangeResult
|
||||||
err = withStoreLock(dir, func() error {
|
err = withStoreLock(dir, func() error {
|
||||||
|
entries, err := readStore(dir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if place > len(entries) {
|
||||||
|
return fmt.Errorf("no entry %d: the clipboard history holds %d", place, len(entries))
|
||||||
|
}
|
||||||
|
if entries[place-1].At != at {
|
||||||
|
return fmt.Errorf("entry %d was copied at %s, not at %s: the history changed since it was listed; list it again", place, entries[place-1].At, at)
|
||||||
|
}
|
||||||
|
line := entries[place-1].line
|
||||||
raw, err := os.ReadFile(filepath.Join(dir, "line_cache"))
|
raw, err := os.ReadFile(filepath.Join(dir, "line_cache"))
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
return err
|
return err
|
||||||
@@ -319,7 +358,7 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
_, text, _ := strings.Cut(l, " ")
|
_, text, _ := strings.Cut(l, " ")
|
||||||
if text == line || (id != "" && entryID(text) == id) {
|
if text == line {
|
||||||
out.Removed++
|
out.Removed++
|
||||||
_ = os.Remove(filepath.Join(dir, entryID(text)))
|
_ = os.Remove(filepath.Join(dir, entryID(text)))
|
||||||
continue
|
continue
|
||||||
@@ -327,7 +366,7 @@ func Delete(id, line string) (ChangeResult, error) {
|
|||||||
kept = append(kept, l)
|
kept = append(kept, l)
|
||||||
}
|
}
|
||||||
if out.Removed == 0 {
|
if out.Removed == 0 {
|
||||||
return fmt.Errorf("no entry %s%s in the clipboard history", id, line)
|
return fmt.Errorf("no entry %d in the clipboard history", place)
|
||||||
}
|
}
|
||||||
content := strings.Join(kept, "\n")
|
content := strings.Join(kept, "\n")
|
||||||
if content != "" {
|
if content != "" {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@@ -51,46 +52,84 @@ func store(t *testing.T, clips map[string]string, order ...string) string {
|
|||||||
return dir
|
return dir
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheHistoryIsNewestFirstOnceEachWithItsText(t *testing.T) {
|
func TestTheHistoryAnswersCountTimesAndLengthsNeverContent(t *testing.T) {
|
||||||
store(t, map[string]string{"first": "first", "second (2 lines)": "second\nline two"}, "first", "second (2 lines)", "first")
|
store(t, map[string]string{"first": "first", "sk-ant-api03-SECRETKEY (2 lines)": "sk-ant-api03-SECRETKEY\nline two"}, "first", "sk-ant-api03-SECRETKEY (2 lines)", "first")
|
||||||
fakeProcess(t, nobody, "clipmenud")
|
fakeProcess(t, nobody, "clipmenud")
|
||||||
h, err := History(10, 4096)
|
h, err := History(10)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if !h.Collecting || h.Total != 2 || h.Entries[0].Line != "first" || h.Entries[1].Text != "second\nline two" || h.Entries[1].Bytes != 15 {
|
if !h.Collecting || h.Total != 2 || len(h.Entries) != 2 || h.Entries[0].Bytes != 5 || h.Entries[1].Bytes != 31 || h.Entries[0].At == "" {
|
||||||
t.Fatalf("%+v", h)
|
t.Fatalf("%+v", h)
|
||||||
}
|
}
|
||||||
if h, _ := History(1, 3); len(h.Entries) != 1 || h.Entries[0].Text != "fir" || !h.Entries[0].Truncated {
|
raw, _ := json.Marshal(h)
|
||||||
t.Fatalf("limited and cut: %+v", h)
|
for _, leak := range []string{"first", "SECRETKEY", "line two"} {
|
||||||
|
if strings.Contains(string(raw), leak) {
|
||||||
|
t.Fatalf("the history answers %q: %s", leak, raw)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if h, _ := History(10, 0); h.Entries[0].Text != "" || h.Entries[0].Bytes != 5 {
|
if h, _ := History(1); len(h.Entries) != 1 {
|
||||||
t.Fatalf("without text: %+v", h)
|
t.Fatalf("limited: %+v", h)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnEntryIsDeletedByIdOrLineWithItsText(t *testing.T) {
|
// at is the time History reports for the entry at place.
|
||||||
|
func at(t *testing.T, place int) string {
|
||||||
|
t.Helper()
|
||||||
|
h, err := History(100)
|
||||||
|
if err != nil || place > len(h.Entries) {
|
||||||
|
t.Fatalf("%+v, %v", h, err)
|
||||||
|
}
|
||||||
|
return h.Entries[place-1].At
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADeleteWhoseListingIsStaleIsRefused(t *testing.T) {
|
||||||
|
dir := store(t, map[string]string{"a": "a", "b": "b"}, "a", "b")
|
||||||
|
listed := at(t, 1) // b
|
||||||
|
f, _ := os.OpenFile(filepath.Join(dir, "line_cache"), os.O_APPEND|os.O_WRONLY, 0o600)
|
||||||
|
f.WriteString("1800000000000000000 a\n") // a copy arrives: place 1 is now a
|
||||||
|
f.Close()
|
||||||
|
if _, err := Delete(1, listed); err == nil || !strings.Contains(err.Error(), "list it again") {
|
||||||
|
t.Fatalf("the wrong entry was deleted: %v", err)
|
||||||
|
}
|
||||||
|
if h, _ := History(10); h.Total != 2 {
|
||||||
|
t.Fatalf("%+v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEntryIsDeletedByItsPlaceWithItsTextAndNeverByItsText(t *testing.T) {
|
||||||
dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a")
|
dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a")
|
||||||
r, err := Delete(entryID("a"), "")
|
// newest first: a, c, b
|
||||||
|
r, err := Delete(1, at(t, 1))
|
||||||
if err != nil || r.Removed != 2 || r.Remaining != 2 {
|
if err != nil || r.Removed != 2 || r.Remaining != 2 {
|
||||||
t.Fatalf("by id, both copies: %+v, %v", r, err)
|
t.Fatalf("by place, both copies: %+v, %v", r, err)
|
||||||
}
|
}
|
||||||
if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) {
|
if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) {
|
||||||
t.Fatal("the text stayed")
|
t.Fatal("the text stayed")
|
||||||
}
|
}
|
||||||
if r, err := Delete("", "b"); err != nil || r.Removed != 1 || r.Remaining != 1 {
|
if r, err := Delete(2, at(t, 2)); err != nil || r.Removed != 1 || r.Remaining != 1 {
|
||||||
t.Fatalf("by line: %+v, %v", r, err)
|
t.Fatalf("second place (b): %+v, %v", r, err)
|
||||||
}
|
}
|
||||||
if _, err := Delete("", "zzz"); err == nil {
|
if _, err := Delete(5, "x"); err == nil {
|
||||||
t.Fatal("a missing entry was reported deleted")
|
t.Fatal("a missing place was reported deleted")
|
||||||
}
|
}
|
||||||
if _, err := Delete("x", "y"); err == nil {
|
if _, err := Delete(0, "x"); err == nil {
|
||||||
t.Fatal("both an id and a line were accepted")
|
t.Fatal("place 0 was accepted")
|
||||||
}
|
}
|
||||||
cache, _ := os.ReadFile(filepath.Join(dir, "line_cache"))
|
cache, _ := os.ReadFile(filepath.Join(dir, "line_cache"))
|
||||||
if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 {
|
if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 {
|
||||||
t.Fatalf("line cache: %q", cache)
|
t.Fatalf("line cache: %q", cache)
|
||||||
}
|
}
|
||||||
|
for _, tool := range tools() {
|
||||||
|
if tool.Name == "clipmenu_delete" {
|
||||||
|
if _, ok := tool.Input["line"]; ok {
|
||||||
|
t.Fatal("delete still takes a line")
|
||||||
|
}
|
||||||
|
if _, ok := tool.Input["id"]; ok {
|
||||||
|
t.Fatal("delete still takes an id")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
||||||
@@ -112,13 +151,21 @@ func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) {
|
func copyBed(t *testing.T, xsel string) string {
|
||||||
|
t.Helper()
|
||||||
store(t, nil)
|
store(t, nil)
|
||||||
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
||||||
bin := fakeBinaries(t, map[string]string{"systemctl": "true", "systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`})
|
bin := fakeBinaries(t, map[string]string{"systemctl": "true", "xsel": xsel,
|
||||||
|
"systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`})
|
||||||
t.Setenv("LOG", filepath.Join(bin, "log"))
|
t.Setenv("LOG", filepath.Join(bin, "log"))
|
||||||
|
readBackWait = 0
|
||||||
|
return bin
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManagerAndReadsItBack(t *testing.T) {
|
||||||
|
bin := copyBed(t, `cat "$LOG.text"`)
|
||||||
r, err := Copy("secret-free text")
|
r, err := Copy("secret-free text")
|
||||||
if err != nil || r.Bytes != 16 || !strings.HasPrefix(r.Unit, "clipmenu-copy-") {
|
if err != nil || r.Bytes != 16 || !r.Verified || !strings.HasPrefix(r.Unit, "clipmenu-copy-") {
|
||||||
t.Fatalf("%+v, %v", r, err)
|
t.Fatalf("%+v, %v", r, err)
|
||||||
}
|
}
|
||||||
asked, _ := os.ReadFile(filepath.Join(bin, "log"))
|
asked, _ := os.ReadFile(filepath.Join(bin, "log"))
|
||||||
@@ -134,7 +181,23 @@ func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) {
|
func TestCopyFailsWhenTheClipboardHoldsSomethingElse(t *testing.T) {
|
||||||
|
copyBed(t, `printf 'old contents'`)
|
||||||
|
_, err := Copy("new text")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "does not hold") || strings.Contains(err.Error(), "old contents") {
|
||||||
|
t.Fatalf("a copy that did not land was reported: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyFailsSayingWhyWhenTheOwnerCannotReachTheDisplay(t *testing.T) {
|
||||||
|
copyBed(t, `echo "xsel: Can't open display: (null)" >&2; exit 1`)
|
||||||
|
_, err := Copy("new text")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "Can't open display") {
|
||||||
|
t.Fatalf("the reason was not given: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPasteSaysHowMuchTheClipboardHoldsNeverWhat(t *testing.T) {
|
||||||
fakeMachine(t)
|
fakeMachine(t)
|
||||||
if _, err := Paste(); !errors.Is(err, ErrNoSession) {
|
if _, err := Paste(); !errors.Is(err, ErrNoSession) {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -142,8 +205,9 @@ func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) {
|
|||||||
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
fakeProcess(t, nobody, "i3", "DISPLAY=:1")
|
||||||
fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`})
|
fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`})
|
||||||
p, err := Paste()
|
p, err := Paste()
|
||||||
if err != nil || p.Text != "on the clipboard" || p.Bytes != 16 {
|
raw, _ := json.Marshal(p)
|
||||||
t.Fatalf("%+v, %v", p, err)
|
if err != nil || p.Bytes != 16 || strings.Contains(string(raw), "on the clipboard") {
|
||||||
|
t.Fatalf("%s, %v", raw, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,7 +221,26 @@ func TestWithoutAStoreTheChangesSayWhy(t *testing.T) {
|
|||||||
if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") {
|
if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if h, err := History(5, 0); err != nil || h.Total != 0 || h.Collecting {
|
if h, err := History(5); err != nil || h.Total != 0 || h.Collecting {
|
||||||
t.Fatalf("an empty history: %+v, %v", h, err)
|
t.Fatalf("an empty history: %+v, %v", h, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// clipmenud is a bash script: its process is named "bash" and only its command line names clipmenud
|
||||||
|
// (issue 860: collecting was false while the daemon ran).
|
||||||
|
func TestCollectingSeesClipmenudRunAsABashScript(t *testing.T) {
|
||||||
|
store(t, map[string]string{"first": "first"}, "first")
|
||||||
|
fakeProcess(t, nobody, "bash")
|
||||||
|
cmdline := filepath.Join(procRoot, strconv.Itoa(nobody), "cmdline")
|
||||||
|
if err := os.WriteFile(cmdline, []byte("bash\x00/usr/bin/clipmenud\x00"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if h, err := History(10); err != nil || !h.Collecting {
|
||||||
|
t.Fatalf("clipmenud runs as a bash script: %+v %v", h, err)
|
||||||
|
}
|
||||||
|
// another bash script is not the daemon
|
||||||
|
os.WriteFile(cmdline, []byte("bash\x00/usr/bin/other\x00"), 0o644)
|
||||||
|
if h, _ := History(10); h.Collecting {
|
||||||
|
t.Fatalf("a bash script that is not clipmenud counted: %+v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -22,28 +22,23 @@ func tools() []stdio.Tool {
|
|||||||
return []stdio.Tool{
|
return []stdio.Tool{
|
||||||
{
|
{
|
||||||
Name: "node-clipboard.history",
|
Name: "node-clipboard.history",
|
||||||
Description: "What the operator copied, newest first: each entry's id, its first line, when, its size " +
|
Description: "What the operator copied, newest first: how many entries, when each was copied and how long " +
|
||||||
"and its text (each cut at max_bytes). Whether the clipboard daemon is collecting.",
|
"it is, never what it says. Whether the clipboard daemon is collecting. The text is read on the operator's own screen, through clipmenu.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"},
|
"limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"},
|
||||||
"max_bytes": map[string]any{"type": "integer", "description": "cut each entry's text at this many bytes; 0 leaves the text out (default 4096, at most 65536)"},
|
|
||||||
},
|
},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
limit, err := whole(args, "limit", 20, 1, 500)
|
limit, err := whole(args, "limit", 20, 1, 500)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
most, err := whole(args, "max_bytes", 4096, 0, 65536)
|
return History(limit)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return History(limit, most)
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "node-clipboard.copy",
|
Name: "node-clipboard.copy",
|
||||||
Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " +
|
Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " +
|
||||||
"like any copy. Answers how many bytes.",
|
"like any copy. Reads the clipboard back and answers an error with the reason when it does not hold the text.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"properties": map[string]any{
|
"properties": map[string]any{
|
||||||
@@ -61,7 +56,7 @@ func tools() []stdio.Tool {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "clipmenu_paste",
|
Name: "clipmenu_paste",
|
||||||
Description: "What the operator's clipboard holds right now, as text (cut at 64 KiB, said in truncated).",
|
Description: "How many bytes of text the operator's clipboard holds right now, never the text itself.",
|
||||||
Run: func(map[string]any) (any, error) { return Paste() },
|
Run: func(map[string]any) (any, error) { return Paste() },
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -71,19 +66,22 @@ func tools() []stdio.Tool {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "clipmenu_delete",
|
Name: "clipmenu_delete",
|
||||||
Description: "Forget one entry of the clipboard history, by its id as the history answers it, or by " +
|
Description: "Forget one entry of the clipboard history, by its place in node-clipboard.history's list " +
|
||||||
"its first line exactly.",
|
"(1 is the newest) and its time as the history gives it; refused when the two no longer match.",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"id": map[string]any{"type": "string", "description": "the entry's id"},
|
"place": map[string]any{"type": "integer", "description": "the entry's place in the history, 1 for the newest"},
|
||||||
"line": map[string]any{"type": "string", "description": "the entry's first line, exactly"},
|
"at": map[string]any{"type": "string", "description": "the entry's time, as the history reports it"},
|
||||||
},
|
},
|
||||||
Run: func(args map[string]any) (any, error) {
|
Run: func(args map[string]any) (any, error) {
|
||||||
id, err := text(args, "id", false)
|
place, err := whole(args, "place", 0, 0, 100000)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
line, _ := args["line"].(string)
|
at, err := text(args, "at", true)
|
||||||
return Delete(id, line)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return Delete(place, at)
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,16 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history
|
// clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history
|
||||||
@@ -27,20 +34,48 @@ func TestItClaimsTheClipboardSeatServingHistoryAndCopy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTheDaemonStartsOnceFromTheSessionsStart(t *testing.T) {
|
func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) {
|
||||||
m := readManifest(t)
|
m := readManifest(t)
|
||||||
if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" {
|
if len(m.Shell) != 0 {
|
||||||
t.Fatalf("%+v", m.Shell)
|
t.Fatalf("a second start from the session: %+v", m.Shell)
|
||||||
}
|
}
|
||||||
code := m.Shell[0].Code
|
svc := m.resource(t, "daemon")
|
||||||
// Started once, through the module's text-only xsel (a selection holding an image is never read).
|
want := map[string]any{"id": "daemon", "type": "service", "unit": "clipmenud.service", "scope": "user",
|
||||||
if strings.Count(code, "clipmenud &\n") != 1 || !strings.Contains(code, `PATH="/usr/local/lib/mesh-clipmenu:$PATH" clipmenud &`) ||
|
"user": "${machine:account}", "state": "running", "boot": "enabled", "reload-on": nil}
|
||||||
strings.Contains(code, "greenclip") {
|
for k, v := range want {
|
||||||
t.Fatalf("%q", code)
|
if k != "reload-on" && svc[k] != v {
|
||||||
|
t.Fatalf("daemon %s = %v, want %v: %v", k, svc[k], v, svc)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, r := range m.Resources {
|
// ADR 0240 rule 8: a long-running resource says how it is ready. The launcher keeps the unit active while
|
||||||
if r["type"] == "service" || r["type"] == "process" {
|
// it waits for the display and execs clipmenud, so the unit's state is the daemon's own liveness.
|
||||||
t.Fatalf("a second start: %v", r)
|
if h, _ := svc["health"].(map[string]any); h["kind"] != "unit" {
|
||||||
|
t.Fatalf("daemon health = %v, want kind unit", svc["health"])
|
||||||
|
}
|
||||||
|
dropin := m.resource(t, "daemon-dropin")
|
||||||
|
content, _ := dropin["content"].(string)
|
||||||
|
if dropin["path"] != "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf" ||
|
||||||
|
dropin["owner"] != "${machine:account}" ||
|
||||||
|
!strings.Contains(content, "ExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session") ||
|
||||||
|
strings.Contains(content, "greenclip") {
|
||||||
|
t.Fatalf("drop-in: %v", dropin)
|
||||||
|
}
|
||||||
|
launcher, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
if !strings.Contains(launcher, "DISPLAY") || !strings.Contains(launcher, "/tmp/.X11-unix") ||
|
||||||
|
!strings.Contains(launcher, `PATH="/usr/local/lib/mesh-clipmenu:$PATH"`) || !strings.Contains(launcher, "exec clipmenud") {
|
||||||
|
t.Fatalf("launcher: %q", launcher)
|
||||||
|
}
|
||||||
|
// Waits for the display instead of exiting into the start limit; DISPLAY from the manager wins.
|
||||||
|
if !strings.Contains(launcher, "until [ -n \"$DISPLAY\" ]") || !strings.Contains(launcher, "sleep 1") ||
|
||||||
|
strings.Contains(launcher, "exit 1") || strings.Contains(launcher, "exit 0") {
|
||||||
|
t.Fatalf("the launcher must wait, not exit: %q", launcher)
|
||||||
|
}
|
||||||
|
if !strings.Contains(content, "StartLimitIntervalSec=0") || !strings.Contains(content, "RestartSec=2s") {
|
||||||
|
t.Fatalf("drop-in lacks the safety net: %q", content)
|
||||||
|
}
|
||||||
|
for k, v := range m.Environment.Variables {
|
||||||
|
if !strings.Contains(content, "Environment="+k+"="+v) {
|
||||||
|
t.Fatalf("the unit lacks %s=%s", k, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -64,3 +99,126 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
|
|||||||
checkTheToolsAgree(t, m)
|
checkTheToolsAgree(t, m)
|
||||||
checkNoSecretsOrInstallationNames(t)
|
checkNoSecretsOrInstallationNames(t)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Run for real: DISPLAY from the manager is used at once; several sockets and no DISPLAY wait.
|
||||||
|
func TestTheLauncherUsesTheManagersDisplayAndWaitsOtherwise(t *testing.T) {
|
||||||
|
m := readManifest(t)
|
||||||
|
script, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := dir + "/launcher"
|
||||||
|
// clipmenud and the socket directory are faked: the script's own paths are rewritten.
|
||||||
|
body := strings.ReplaceAll(script, "/tmp/.X11-unix", dir+"/x11")
|
||||||
|
body = strings.Replace(body, "exec clipmenud", `echo "DISPLAY=$DISPLAY"`, 1)
|
||||||
|
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.MkdirAll(dir+"/x11", 0o755)
|
||||||
|
fakeBin := dir + "/bin"
|
||||||
|
os.MkdirAll(fakeBin, 0o755)
|
||||||
|
// the manager's environment: empty until the file appears
|
||||||
|
os.WriteFile(fakeBin+"/systemctl", []byte("#!/bin/sh\n[ -f \""+dir+"/env\" ] && cat \""+dir+"/env\"\nexit 0\n"), 0o755)
|
||||||
|
run := func(env ...string) (string, bool) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, "sh", file)
|
||||||
|
cmd.Env = append([]string{"PATH=" + fakeBin + ":" + os.Getenv("PATH"), "HOME=" + dir}, env...)
|
||||||
|
out, _ := cmd.Output()
|
||||||
|
return strings.TrimSpace(string(out)), ctx.Err() == nil
|
||||||
|
}
|
||||||
|
if out, ok := run("DISPLAY=:7"); !ok || out != "DISPLAY=:7" {
|
||||||
|
t.Fatalf("with DISPLAY: %q %v", out, ok)
|
||||||
|
}
|
||||||
|
if _, ok := run(); ok {
|
||||||
|
t.Fatal("it exited with no X server")
|
||||||
|
}
|
||||||
|
for _, n := range []string{"X0", "X1"} {
|
||||||
|
l, _ := net.Listen("unix", dir+"/x11/"+n)
|
||||||
|
defer l.Close()
|
||||||
|
}
|
||||||
|
if _, ok := run(); ok {
|
||||||
|
t.Fatal("it chose one of two servers")
|
||||||
|
}
|
||||||
|
// the session hands the manager its display while the launcher waits
|
||||||
|
go func() {
|
||||||
|
time.Sleep(1200 * time.Millisecond)
|
||||||
|
os.WriteFile(dir+"/env", []byte("DISPLAY=:1\nOTHER=x\n"), 0o644)
|
||||||
|
}()
|
||||||
|
if out, ok := run(); !ok || out != "DISPLAY=:1" {
|
||||||
|
t.Fatalf("the display that arrived while waiting: %q %v", out, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run for real: while another clipmenud of the account holds the session lock the launcher waits without
|
||||||
|
// a word, and starts clipmenud once the lock is free (issue 862).
|
||||||
|
func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T) {
|
||||||
|
if _, err := exec.LookPath("flock"); err != nil {
|
||||||
|
t.Skip("flock is missing here")
|
||||||
|
}
|
||||||
|
m := readManifest(t)
|
||||||
|
script, _ := m.resource(t, "launcher")["content"].(string)
|
||||||
|
dir := t.TempDir()
|
||||||
|
body := strings.Replace(script, "exec clipmenud", `echo "started"`, 1)
|
||||||
|
file := dir + "/launcher"
|
||||||
|
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
store := dir + "/cm/clipmenu.6.op"
|
||||||
|
os.MkdirAll(store, 0o700)
|
||||||
|
lock := store + "/session_lock"
|
||||||
|
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed
|
||||||
|
holder := exec.Command("flock", "-x", "-n", lock, "sleep", "60")
|
||||||
|
holder.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||||
|
if err := holder.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// flock's child shares the lock, so the whole group goes
|
||||||
|
release := func() { syscall.Kill(-holder.Process.Pid, syscall.SIGKILL); holder.Wait() }
|
||||||
|
defer release()
|
||||||
|
for i := 0; i < 50; i++ {
|
||||||
|
if probe := exec.Command("flock", "-n", lock, "true"); probe.Run() != nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
var out syncBuffer
|
||||||
|
cmd := exec.Command("sh", file)
|
||||||
|
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "DISPLAY=:7", "USER=op", "CM_DIR=" + dir + "/cm"}
|
||||||
|
cmd.Stdout, cmd.Stderr = &out, &out
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- cmd.Wait() }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
t.Fatalf("the launcher went on while the lock was held: %q", out.String())
|
||||||
|
case <-time.After(2500 * time.Millisecond):
|
||||||
|
}
|
||||||
|
if out.Len() != 0 {
|
||||||
|
t.Fatalf("the launcher spoke while it waited: %q", out.String())
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
cmd.Process.Kill()
|
||||||
|
t.Fatal("the launcher did not start clipmenud after the lock was freed")
|
||||||
|
}
|
||||||
|
if got := strings.TrimSpace(out.String()); got != "started" {
|
||||||
|
t.Fatalf("after the lock was freed: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type syncBuffer struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
b strings.Builder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *syncBuffer) Write(p []byte) (int, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.b.Write(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *syncBuffer) Len() int { s.mu.Lock(); defer s.mu.Unlock(); return s.b.Len() }
|
||||||
|
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
|
||||||
|
|||||||
@@ -403,7 +403,7 @@ func processesOf(comm string) []int {
|
|||||||
if owner, ok := ownerOf(dir); !ok || owner != uid {
|
if owner, ok := ownerOf(dir); !ok || owner != uid {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if readTrimmed(filepath.Join(dir, "comm")) == comm {
|
if readTrimmed(filepath.Join(dir, "comm")) == comm || runsScript(dir, comm) {
|
||||||
pids = append(pids, pid)
|
pids = append(pids, pid)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -411,6 +411,24 @@ func processesOf(comm string) []int {
|
|||||||
return pids
|
return pids
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// runsScript is whether the process is an interpreter running the script named name: clipmenud is a
|
||||||
|
// bash script, so its comm is "bash" and only its command line, "bash /usr/bin/clipmenud", names it.
|
||||||
|
func runsScript(procDir, name string) bool {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(procDir, "cmdline"))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
|
||||||
|
if len(args) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch filepath.Base(args[0]) {
|
||||||
|
case "bash", "sh", "dash", "zsh", "env":
|
||||||
|
return filepath.Base(args[1]) == name
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// signalAll sends sig to every process of this uid named comm, and answers the pids it reached.
|
// signalAll sends sig to every process of this uid named comm, and answers the pids it reached.
|
||||||
func signalAll(comm string, sig syscall.Signal) []int {
|
func signalAll(comm string, sig syscall.Signal) []int {
|
||||||
var reached []int
|
var reached []int
|
||||||
|
|||||||
@@ -29,13 +29,6 @@
|
|||||||
"CM_HISTLENGTH": "15"
|
"CM_HISTLENGTH": "15"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"shell": [
|
|
||||||
{
|
|
||||||
"for": "xinitrc",
|
|
||||||
"slot": "normal",
|
|
||||||
"code": "# The clipboard's history (module clipmenu, novox/hq ADR 0208): clipmenud collects every copy from\n# here on, once per session. It keeps the history in the account's runtime directory, so a reboot\n# forgets it, and with it every password that was ever copied.\n# Through the module's xsel, which reads only a selection offering text (see the README).\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\" clipmenud &\n"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"resources": [
|
"resources": [
|
||||||
{
|
{
|
||||||
"id": "package",
|
"id": "package",
|
||||||
@@ -54,6 +47,47 @@
|
|||||||
"path": "/usr/local/lib/mesh-clipmenu/xsel",
|
"path": "/usr/local/lib/mesh-clipmenu/xsel",
|
||||||
"mode": "0755",
|
"mode": "0755",
|
||||||
"content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n"
|
"content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "user-units-dir",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0755"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon-dropins",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0755"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon-dropin",
|
||||||
|
"type": "file",
|
||||||
|
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf",
|
||||||
|
"owner": "${machine:account}",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "# clipmenud in the operator's account (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# The package's unit starts clipmenud with no DISPLAY; the module's launcher finds the display and puts the\n# text-only xsel first on the PATH. The settings repeat the module's environment, which the account's\n# service manager does not read.\n[Unit]\nStartLimitIntervalSec=0\n\n[Service]\nRestartSec=2s\nEnvironment=CM_SELECTIONS=clipboard\nEnvironment=CM_MAX_CLIPS=500\nEnvironment=CM_HISTLENGTH=15\nExecStart=\nExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "launcher",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/usr/local/lib/mesh-clipmenu/clipmenud-session",
|
||||||
|
"mode": "0755",
|
||||||
|
"content": "#!/bin/sh\n# clipmenud as the operator's user unit starts it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# The account's service manager starts before the X server does, and has a DISPLAY only once the session's\n# xinitrc has handed it over (module xorg). So this waits, quietly, until the display is known, and exits\n# only when the session ends with it: a quick exit would run the unit into its start limit. DISPLAY, when\n# the manager has it, wins. Without it, exactly one X socket names the display; none or several (a\n# second server beside the session) wait for the session's own to arrive.\n# A running process does not see what the session hands the manager later, so each pass asks the manager.\nuntil [ -n \"$DISPLAY\" ]; do\n\tfor pair in $(systemctl --user show-environment 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY)='); do\n\t\texport \"$pair\"\n\tdone\n\t[ -n \"$DISPLAY\" ] && break\n\tset -- /tmp/.X11-unix/X*\n\tif [ $# -eq 1 ] && [ -S \"$1\" ]; then\n\t\tDISPLAY=\":${1##*/X}\"\n\t\texport DISPLAY\n\telse\n\t\tsleep 1\n\tfi\ndone\nif [ -z \"$XAUTHORITY\" ] && [ -r \"$HOME/.Xauthority\" ]; then\n\tXAUTHORITY=\"$HOME/.Xauthority\"\n\texport XAUTHORITY\nfi\n# clipmenud allows one daemon per account: it takes an exclusive flock on its session_lock file and exits\n# with status 2 when that is held. After a delivery without a new login, the session's older clipmenud (from\n# its xinitrc) still holds it, and a start would run the unit round its restart loop. So this waits, quietly,\n# until the lock is free: it looks at the same file clipmenud locks (the directory is clipmenud's own rule)\n# and lets go at once. Exit status 1 is \"held\" for util-linux and BusyBox flock alike (no -E: BusyBox lacks it); any other outcome starts clipmenud, which says its own\n# error. Another daemon winning the gap between look and start costs one restart, not a loop.\nlock=\"${CM_DIR:-${XDG_RUNTIME_DIR-${TMPDIR-/tmp}}}/clipmenu.6.$USER/session_lock\"\nwhile [ -e \"$lock\" ]; do\n\tflock -n \"$lock\" true\n\t[ $? -eq 1 ] || break\n\tsleep 2\ndone\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\"\nexport PATH\nexec clipmenud\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "daemon",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "clipmenud.service",
|
||||||
|
"scope": "user",
|
||||||
|
"user": "${machine:account}",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled",
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"build": {
|
"build": {
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time
|
How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time
|
||||||
spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue
|
spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue
|
||||||
depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue
|
depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue
|
||||||
312). It changes nothing and declares no resources, so it can be assigned to every machine.
|
312). It also reads a machine's CPU use by systemd unit and by container, from each cgroup's cpu.stat,
|
||||||
|
so an overload outside every container can be named (novox/hq issue 741). It changes nothing and
|
||||||
|
declares no resources, so it can be assigned to every machine.
|
||||||
|
|
||||||
## Why it exists, and why here
|
## Why it exists, and why here
|
||||||
|
|
||||||
@@ -22,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
|
|||||||
|
|
||||||
## Tools
|
## Tools
|
||||||
|
|
||||||
All four only read.
|
All five only read.
|
||||||
|
|
||||||
| tool | what | replaces |
|
| tool | what | replaces |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
@@ -30,6 +32,7 @@ All four only read.
|
|||||||
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
|
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
|
||||||
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
|
||||||
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
|
||||||
|
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
|
||||||
|
|
||||||
### How each number is read
|
### How each number is read
|
||||||
|
|
||||||
|
|||||||
@@ -68,9 +68,10 @@ func (m *Machine) bottomDevices() map[string]bool {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// unitIO is every unit's storage I/O, by its cgroup's path: the innermost .service or .scope, because
|
// unitPaths is every systemd unit's cgroup path under /sys/fs/cgroup: the innermost .service or
|
||||||
// io.stat counts a cgroup's descendants too and an outer unit would count them twice.
|
// .scope, because a cgroup's own stat file counts its descendants too and an outer unit would count
|
||||||
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
// them twice. Shared by every reader that walks the cgroup tree (unitIO here, unitCPU in cpu.go).
|
||||||
|
func (m *Machine) unitPaths() []string {
|
||||||
base := m.path("/sys/fs/cgroup")
|
base := m.path("/sys/fs/cgroup")
|
||||||
var units []string
|
var units []string
|
||||||
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
|
||||||
@@ -89,11 +90,19 @@ func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
|||||||
outer[dir] = true
|
outer[dir] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out := map[string]cgroupIO{}
|
out := make([]string, 0, len(units))
|
||||||
for _, u := range units {
|
for _, u := range units {
|
||||||
if outer[u] {
|
if !outer[u] {
|
||||||
continue
|
out = append(out, u)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitIO is every unit's storage I/O, by its cgroup's path.
|
||||||
|
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
|
||||||
|
out := map[string]cgroupIO{}
|
||||||
|
for _, u := range m.unitPaths() {
|
||||||
text := m.read("/sys/fs/cgroup/" + u + "/io.stat")
|
text := m.read("/sys/fs/cgroup/" + u + "/io.stat")
|
||||||
if text == "" {
|
if text == "" {
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cgroupCPU is a cgroup's processor time, from its cpu.stat.
|
||||||
|
type cgroupCPU struct {
|
||||||
|
usageUsec uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCPUStat reads a cgroup's cpu.stat ("usage_usec 123\nuser_usec …\nsystem_usec …"), readable by
|
||||||
|
// every account, unlike /proc/<pid>/stat for another account's process.
|
||||||
|
func parseCPUStat(s string) cgroupCPU {
|
||||||
|
var c cgroupCPU
|
||||||
|
for _, line := range strings.Split(s, "\n") {
|
||||||
|
k, v, ok := strings.Cut(line, " ")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if k == "usage_usec" {
|
||||||
|
c.usageUsec, _ = strconv.ParseUint(strings.TrimSpace(v), 10, 64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// unitCPU is every unit's processor time, by its cgroup's path. A cgroup whose cpu.stat cannot be
|
||||||
|
// read is left out, never an error for the whole answer.
|
||||||
|
func (m *Machine) unitCPU() map[string]cgroupCPU {
|
||||||
|
out := map[string]cgroupCPU{}
|
||||||
|
for _, u := range m.unitPaths() {
|
||||||
|
text := m.read("/sys/fs/cgroup/" + u + "/cpu.stat")
|
||||||
|
if text == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[u] = parseCPUStat(text)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// MachineCPU is the machine's CPU in use over the window, from /proc/stat's cpu line. CPUsBusy is
|
||||||
|
// rounded only for display; raw, unexported, is the same count unrounded, for topCPU's share and
|
||||||
|
// unaccounted math (review round 1 of novox/hq issue 741: a small busy time that rounds CPUsBusy to
|
||||||
|
// 0.0 must not zero out every unit's share along with it).
|
||||||
|
type MachineCPU struct {
|
||||||
|
CPUs int `json:"cpus"`
|
||||||
|
BusyPct float64 `json:"busy_pct"` // % of all CPUs together
|
||||||
|
CPUsBusy float64 `json:"cpus_busy"` // as a count of CPUs, e.g. 7.5 of 12, rounded for display
|
||||||
|
raw float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineCPUOver is the machine's CPU in use between two readings of /proc/stat: everything but idle
|
||||||
|
// and iowait, counted against the window's total ticks over every CPU.
|
||||||
|
func machineCPUOver(a, b cpuTimes) MachineCPU {
|
||||||
|
total := float64(b.total() - a.total())
|
||||||
|
if total <= 0 {
|
||||||
|
return MachineCPU{CPUs: b.cpus}
|
||||||
|
}
|
||||||
|
idle := sub(b.idle, a.idle) + sub(b.iowait, a.iowait)
|
||||||
|
busy := total - idle
|
||||||
|
if busy < 0 {
|
||||||
|
busy = 0
|
||||||
|
}
|
||||||
|
raw := busy / total * float64(b.cpus)
|
||||||
|
return MachineCPU{CPUs: b.cpus, BusyPct: round1(busy / total * 100), CPUsBusy: round1(raw), raw: raw}
|
||||||
|
}
|
||||||
|
|
||||||
|
// CPUUsage is a unit's or a container's share of CPU between two readings of its cgroup's cpu.stat.
|
||||||
|
// Cgroup is its cgroup's path, relative to /sys/fs/cgroup, said beside Unit always (review round 1 of
|
||||||
|
// novox/hq issue 741): two cgroups can share a leaf name (an outer "init.scope" and a user session's
|
||||||
|
// "user.slice/user@1000.service/init.scope"), and the path is how they are told apart.
|
||||||
|
type CPUUsage struct {
|
||||||
|
Unit string `json:"unit,omitempty"`
|
||||||
|
Cgroup string `json:"cgroup,omitempty"`
|
||||||
|
Container string `json:"container,omitempty"`
|
||||||
|
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
|
||||||
|
SharePct float64 `json:"share_pct"` // this unit's share of the machine's busy time
|
||||||
|
raw float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// topCPU ranks the units, or the containers, with the most CPU time between two readings of
|
||||||
|
// cpu.stat, busiest first. unaccountedPct is the machine's busy time no unit accounts for, as a
|
||||||
|
// share of it (never below 0): the sum is always taken over every unit, whichever by ranks by, so a
|
||||||
|
// limit on the list never changes it.
|
||||||
|
func (m *Machine) topCPU(a, b map[string]cgroupCPU, window time.Duration, by string, limit int, cpusBusy float64) ([]CPUUsage, float64) {
|
||||||
|
windowUsec := window.Seconds() * 1e6
|
||||||
|
type entry struct {
|
||||||
|
path, unit, container string
|
||||||
|
pct float64
|
||||||
|
}
|
||||||
|
var entries []entry
|
||||||
|
var names map[string]string
|
||||||
|
var sumPct float64
|
||||||
|
for path, end := range b {
|
||||||
|
start, ok := a[path]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
d := sub(end.usageUsec, start.usageUsec)
|
||||||
|
if d == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
pct := d / windowUsec * 100
|
||||||
|
sumPct += pct
|
||||||
|
unit := filepath.Base(path)
|
||||||
|
var container string
|
||||||
|
if id := containerID.FindStringSubmatch("/" + path); id != nil {
|
||||||
|
if names == nil {
|
||||||
|
names = m.containerNames()
|
||||||
|
}
|
||||||
|
container = id[1][:12]
|
||||||
|
if n, ok := names[id[1]]; ok {
|
||||||
|
container = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries = append(entries, entry{path: path, unit: unit, container: container, pct: pct})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Units are keyed by their cgroup's own path, never by its leaf name (filepath.Base): two cgroups can
|
||||||
|
// share a leaf name (an outer "init.scope" and a user session's nested one), and keying by the name
|
||||||
|
// alone would merge them into one row (review round 1 of novox/hq issue 741). Containers stay keyed
|
||||||
|
// by their name, as before.
|
||||||
|
groups := map[string]*CPUUsage{}
|
||||||
|
for _, e := range entries {
|
||||||
|
key := e.path
|
||||||
|
if by == "container" {
|
||||||
|
if e.container == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key = e.container
|
||||||
|
}
|
||||||
|
g, ok := groups[key]
|
||||||
|
if !ok {
|
||||||
|
g = &CPUUsage{Unit: e.unit, Cgroup: e.path, Container: e.container}
|
||||||
|
if by == "container" {
|
||||||
|
g.Unit, g.Cgroup = "", ""
|
||||||
|
}
|
||||||
|
groups[key] = g
|
||||||
|
}
|
||||||
|
g.raw += e.pct
|
||||||
|
}
|
||||||
|
out := make([]CPUUsage, 0, len(groups))
|
||||||
|
for _, g := range groups {
|
||||||
|
out = append(out, *g)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].raw != out[j].raw {
|
||||||
|
return out[i].raw > out[j].raw
|
||||||
|
}
|
||||||
|
return out[i].Unit+out[i].Cgroup+out[i].Container < out[j].Unit+out[j].Cgroup+out[j].Container
|
||||||
|
})
|
||||||
|
if len(out) > limit {
|
||||||
|
out = out[:limit]
|
||||||
|
}
|
||||||
|
for i := range out {
|
||||||
|
out[i].CPUPct = round1(out[i].raw)
|
||||||
|
if cpusBusy > 0 {
|
||||||
|
out[i].SharePct = round1(out[i].raw / (cpusBusy * 100) * 100)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unaccounted := cpusBusy*100 - sumPct
|
||||||
|
if unaccounted < 0 {
|
||||||
|
unaccounted = 0
|
||||||
|
}
|
||||||
|
unaccountedPct := 0.0
|
||||||
|
if cpusBusy > 0 {
|
||||||
|
unaccountedPct = round1(unaccounted / (cpusBusy * 100) * 100)
|
||||||
|
}
|
||||||
|
return out, unaccountedPct
|
||||||
|
}
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit,
|
||||||
|
// and a container, each with a known usage_usec delta, leaving part of the machine's busy time
|
||||||
|
// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers.
|
||||||
|
func aCPUHog(t *testing.T) *fake {
|
||||||
|
f := newFake(t)
|
||||||
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n")
|
||||||
|
f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n")
|
||||||
|
|
||||||
|
zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n"
|
||||||
|
cg := func(path string, usec int) {
|
||||||
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero)
|
||||||
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n")
|
||||||
|
}
|
||||||
|
cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s
|
||||||
|
cg("system.slice/beta.service", 400000) // 20 %
|
||||||
|
cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime
|
||||||
|
f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n"
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
if m["by"] != "unit" || f.waited != 2*time.Second {
|
||||||
|
t.Errorf("the default: %+v, waited %s", m, f.waited)
|
||||||
|
}
|
||||||
|
cpu := m["cpu"].(MachineCPU)
|
||||||
|
if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 {
|
||||||
|
t.Errorf("machine cpu %+v", cpu)
|
||||||
|
}
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 3 {
|
||||||
|
t.Fatalf("three units moved: %+v", top)
|
||||||
|
}
|
||||||
|
if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 {
|
||||||
|
t.Errorf("busiest first: %+v", top[0])
|
||||||
|
}
|
||||||
|
if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 {
|
||||||
|
t.Errorf("the container, named by the runtime: %+v", top[1])
|
||||||
|
}
|
||||||
|
if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 {
|
||||||
|
t.Errorf("the least busy: %+v", top[2])
|
||||||
|
}
|
||||||
|
if m["unaccounted_pct"] != 33.3 {
|
||||||
|
t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) {
|
||||||
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 {
|
||||||
|
t.Errorf("by container: %+v", top)
|
||||||
|
}
|
||||||
|
if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil {
|
||||||
|
t.Error("by=process was accepted; cpu_top only ranks units and containers")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopLimitsHowManyAreNamed(t *testing.T) {
|
||||||
|
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Unit != "alpha.service" {
|
||||||
|
t.Errorf("limit 1: %+v", top)
|
||||||
|
}
|
||||||
|
// The limit narrows the list, never the unaccounted share, which is taken over every unit.
|
||||||
|
if m["unaccounted_pct"] != 33.3 {
|
||||||
|
t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n")
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
for _, u := range top {
|
||||||
|
if u.Unit == "short-lived.service" {
|
||||||
|
t.Errorf("a unit only in the second reading was counted: %+v", top)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(f.calls, ","), "docker ps") {
|
||||||
|
t.Errorf("the container runtime was never asked: %v", f.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aTinyLoad is a machine with 12 CPUs, only 0.04 of them busy over a 2 s window (busy 1 of 300 total
|
||||||
|
// ticks): rounded to one decimal for display, CPUsBusy reads 0.0, but a unit using half of that busy
|
||||||
|
// time still has a non-zero share — review round 1's decision that share_pct and unaccounted_pct are
|
||||||
|
// computed from the unrounded busy time, never from the rounded CPUsBusy.
|
||||||
|
func aTinyLoad(t *testing.T) *fake {
|
||||||
|
f := newFake(t)
|
||||||
|
cpus := "cpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0 0 0\n" +
|
||||||
|
"cpu4 0 0 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0 0 0\n" +
|
||||||
|
"cpu8 0 0 0 0 0 0 0 0 0 0\ncpu9 0 0 0 0 0 0 0 0 0 0\ncpu10 0 0 0 0 0 0 0 0 0 0\ncpu11 0 0 0 0 0 0 0 0 0 0\n"
|
||||||
|
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\n"+cpus)
|
||||||
|
f.then("/proc/stat", "cpu 1 0 0 299 0 0 0 0 0 0\n"+cpus)
|
||||||
|
|
||||||
|
f.file("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 0\n")
|
||||||
|
f.then("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 60000\n") // 3 % of one CPU over 2 s
|
||||||
|
return f
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCPUTopSharesAreNonZeroEvenWhenRoundedCPUsBusyIsZero(t *testing.T) {
|
||||||
|
out, err := tool(t, aTinyLoad(t).machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m := out.(map[string]any)
|
||||||
|
cpu := m["cpu"].(MachineCPU)
|
||||||
|
if cpu.CPUs != 12 || cpu.CPUsBusy != 0 {
|
||||||
|
t.Fatalf("the machine 0.04 CPUs busy, rounded for display: %+v", cpu)
|
||||||
|
}
|
||||||
|
top := m["top"].([]CPUUsage)
|
||||||
|
if len(top) != 1 || top[0].Unit != "tiny.service" || top[0].CPUPct != 3 || top[0].SharePct != 75 {
|
||||||
|
t.Fatalf("a non-zero share from the unrounded busy time: %+v", top)
|
||||||
|
}
|
||||||
|
if m["unaccounted_pct"] != 25.0 {
|
||||||
|
t.Fatalf("unaccounted from the unrounded busy time: %+v (want 25, so shares plus unaccounted add to 100)",
|
||||||
|
m["unaccounted_pct"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows is review round 1's other decision: units are keyed
|
||||||
|
// by their cgroup path, never by filepath.Base, so two cgroups that both end in "init.scope" — one at the
|
||||||
|
// root, one under a user's session — are two rows, not one merged by their shared leaf name.
|
||||||
|
func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
|
||||||
|
f := aCPUHog(t)
|
||||||
|
cg := func(path string, usec int) {
|
||||||
|
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec 0\n")
|
||||||
|
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\n")
|
||||||
|
}
|
||||||
|
cg("init.scope", 1000000) // the root init, 50 % of one CPU over 2 s
|
||||||
|
cg("user.slice/user@1000.service/init.scope", 400000) // a user session's init, 20 %
|
||||||
|
|
||||||
|
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0, "limit": 10.0})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
top := out.(map[string]any)["top"].([]CPUUsage)
|
||||||
|
var inits []CPUUsage
|
||||||
|
for _, u := range top {
|
||||||
|
if u.Unit == "init.scope" {
|
||||||
|
inits = append(inits, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(inits) != 2 {
|
||||||
|
t.Fatalf("two cgroups named init.scope, merged into %d row(s): %+v", len(inits), top)
|
||||||
|
}
|
||||||
|
byCgroup := map[string]CPUUsage{}
|
||||||
|
for _, u := range inits {
|
||||||
|
byCgroup[u.Cgroup] = u
|
||||||
|
}
|
||||||
|
if u, ok := byCgroup["init.scope"]; !ok || u.CPUPct != 50 {
|
||||||
|
t.Errorf("the root init.scope: %+v", byCgroup)
|
||||||
|
}
|
||||||
|
if u, ok := byCgroup["user.slice/user@1000.service/init.scope"]; !ok || u.CPUPct != 20 {
|
||||||
|
t.Errorf("the user session's init.scope: %+v", byCgroup)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -85,6 +85,57 @@ func Tools(m *Machine) []stdio.Tool {
|
|||||||
return out, nil
|
return out, nil
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
Name: "cpu_top",
|
||||||
|
Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " +
|
||||||
|
"CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " +
|
||||||
|
"container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " +
|
||||||
|
"by name. Replaces top, systemd-cgtop. (r)",
|
||||||
|
Input: map[string]any{
|
||||||
|
"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"},
|
||||||
|
"limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"},
|
||||||
|
"by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"},
|
||||||
|
},
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
n, err := count(args, "seconds", 3, 1, 10)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
w := time.Duration(n) * time.Second
|
||||||
|
limit, err := count(args, "limit", 15, 1, 50)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
by := text(args, "by")
|
||||||
|
switch by {
|
||||||
|
case "":
|
||||||
|
by = "unit"
|
||||||
|
case "unit", "container":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("by is unit or container, not %q", by)
|
||||||
|
}
|
||||||
|
ac, err := parseCPU(m.read("/proc/stat"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
a := m.unitCPU()
|
||||||
|
m.Wait(w)
|
||||||
|
bc, err := parseCPU(m.read("/proc/stat"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b := m.unitCPU()
|
||||||
|
machine := machineCPUOver(ac, bc)
|
||||||
|
top, unaccounted := m.topCPU(a, b, w, by, limit, machine.raw)
|
||||||
|
return map[string]any{
|
||||||
|
"window_s": w.Seconds(),
|
||||||
|
"by": by,
|
||||||
|
"cpu": machine,
|
||||||
|
"top": top,
|
||||||
|
"unaccounted_pct": unaccounted,
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
Name: "disk_filesystems",
|
Name: "disk_filesystems",
|
||||||
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
|
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
|
||||||
|
|||||||
@@ -5,7 +5,8 @@
|
|||||||
"disk_load",
|
"disk_load",
|
||||||
"disk_top",
|
"disk_top",
|
||||||
"disk_filesystems",
|
"disk_filesystems",
|
||||||
"disk_devices"
|
"disk_devices",
|
||||||
|
"cpu_top"
|
||||||
],
|
],
|
||||||
"build": {
|
"build": {
|
||||||
"artifacts": [
|
"artifacts": [
|
||||||
|
|||||||
+22
-5
@@ -44,7 +44,7 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
|
|||||||
|
|
||||||
| tool | | what |
|
| tool | | what |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) or `patch` (a unified diff, as `git diff` writes it); answers the commit and its parent |
|
| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) `patch` (a unified diff, as `git diff` writes it), or `patch_file` with `patch_sha256` (a patch too large to type, see below); answers the commit and its parent |
|
||||||
| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject |
|
| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject |
|
||||||
| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch |
|
| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch |
|
||||||
| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments |
|
| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments |
|
||||||
@@ -52,13 +52,28 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
|
|||||||
| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path |
|
| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path |
|
||||||
|
|
||||||
**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266):
|
**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266):
|
||||||
never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the
|
never the repository's default branch, never a branch a protection rule names. The tools' account may not read the
|
||||||
|
rules themselves (the forge keeps them for a repository's administrator, novox/hq ADR 0365): for it an existing branch
|
||||||
|
is judged by the protection the forge says it applies there, and a branch not made yet by the forge itself, which
|
||||||
|
refuses the commit that would make a branch a rule guards. A rule covers a branch as the
|
||||||
forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob
|
forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob
|
||||||
(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob
|
(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob
|
||||||
the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the
|
the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the
|
||||||
commit. Nothing about merging changes: a merge still needs an independent review at the head and both
|
commit. Nothing about merging changes: a merge still needs an independent review at the head and both
|
||||||
required checks green, through `gitea_merge_pull_request`.
|
required checks green, through `gitea_merge_pull_request`.
|
||||||
|
|
||||||
|
**A patch too large to type** (novox/hq issue 852: a feature's diff is megabytes) goes as `patch_file`, with
|
||||||
|
`patch_sha256`. The caller writes the patch into the module's spool directory `forge-patches`
|
||||||
|
(`/var/lib/mesh/forge-patches`, mode 0733 and owned by the machine's account, which the tools run as: anyone writes,
|
||||||
|
nobody lists; the bundle is given its path as `MESH_GITEA_PATCH_DIR`) and names it: `patch_file` is the file's NAME,
|
||||||
|
32 to 64 characters of `0-9a-f` that the caller chooses at random, never a path. The tool then reads that one file,
|
||||||
|
applies it exactly as it applies `patch` (same size limit, same answer) and removes it once it has found it, whether
|
||||||
|
the commit was made or not. A file refused before that (a bad name, a missing hash or spool directory, a name with no
|
||||||
|
file) stays in the spool. It refuses, sending nothing, a name of another shape, a file that is missing, empty, over
|
||||||
|
4 MiB, with more than one hard link or not a regular file (a symlink is never followed), and a file whose SHA-256 is not `patch_sha256`: the directory has no sticky bit
|
||||||
|
(the manifest allows none), so another writer could replace a file, and the hash is how the caller knows the tool read
|
||||||
|
the bytes it wrote. It reads nothing outside the directory.
|
||||||
|
|
||||||
**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the
|
**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the
|
||||||
ref holds is updated, one it does not is created, and an update or delete carries the blob read just
|
ref holds is updated, one it does not is created, and an update or delete carries the blob read just
|
||||||
before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at
|
before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at
|
||||||
@@ -66,7 +81,7 @@ the commit the branch (or, for a new branch, its base) was read at, never at its
|
|||||||
forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the
|
forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the
|
||||||
file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the
|
file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the
|
||||||
one the forge says it made the commit on; when the branch moved between the read and the commit, the answer
|
one the forge says it made the commit on; when the branch moved between the read and the commit, the answer
|
||||||
says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the
|
says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the tools' account, so its sha is not the
|
||||||
local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a
|
local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a
|
||||||
branch not at the commit the forge named is a failure, never a success.
|
branch not at the commit the forge named is a failure, never a success.
|
||||||
|
|
||||||
@@ -75,8 +90,10 @@ is read once and searched in memory; a binary file is passed over, a file over 4
|
|||||||
`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A
|
`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A
|
||||||
search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`.
|
search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`.
|
||||||
|
|
||||||
It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one
|
It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the
|
||||||
tool call takes at most 50 seconds.
|
tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and
|
||||||
|
keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool
|
||||||
|
call takes at most 50 seconds.
|
||||||
`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on
|
`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on
|
||||||
a new branch from its base; an existing branch committed on; the default branch, a protected branch and
|
a new branch from its base; an existing branch committed on; the default branch, a protected branch and
|
||||||
unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another
|
unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another
|
||||||
|
|||||||
+81
-12
@@ -8,6 +8,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js";
|
|||||||
import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
|
||||||
import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js";
|
import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js";
|
||||||
import { isCommit } from "./checks.js";
|
import { isCommit } from "./checks.js";
|
||||||
|
import { TOOLS_ACCOUNT } from "./tools-account.js";
|
||||||
|
|
||||||
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
|
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
|
||||||
* call, whether the names came from a caller or from the forge's own answer. */
|
* call, whether the names came from a caller or from the forge's own answer. */
|
||||||
@@ -124,10 +125,12 @@ export interface ReadyMark {
|
|||||||
|
|
||||||
/** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which
|
/** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which
|
||||||
* the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file
|
* the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file
|
||||||
* that cannot be read leaves the mesh's own forge account alone, said; never an empty list read as everyone. */
|
* that cannot be read leaves the mesh's own forge accounts alone, said; never an empty list read as everyone: the
|
||||||
|
* account the tools act as (novox/hq ADR 0365) and the admin account, whose marks were made before it. */
|
||||||
export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER,
|
export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER,
|
||||||
read: (f: string) => string = (f) => readFileSync(f, "utf8")): string[] {
|
read: (f: string) => string = (f) => readFileSync(f, "utf8"),
|
||||||
const fallback = own ? [own] : [];
|
tools = process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT): string[] {
|
||||||
|
const fallback = [...new Set([tools, own].filter((a): a is string => !!a))];
|
||||||
if (!file) return fallback;
|
if (!file) return fallback;
|
||||||
try {
|
try {
|
||||||
const line = read(file).split("\n").find((l) => l.startsWith("authors="));
|
const line = read(file).split("\n").find((l) => l.startsWith("authors="));
|
||||||
@@ -444,13 +447,15 @@ export class GiteaClient {
|
|||||||
* fault in the announcer cannot hide itself. Reads only. */
|
* fault in the announcer cannot hide itself. Reads only. */
|
||||||
async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> {
|
async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> {
|
||||||
const out: OpenPull[] = [];
|
const out: OpenPull[] = [];
|
||||||
const rules = new Map<string, BranchProtection[] | undefined>();
|
const rules = new Map<string, BranchProtection | null | undefined>();
|
||||||
for (const found of await this.searchOpenPulls()) {
|
for (const found of await this.searchOpenPulls()) {
|
||||||
const full = `${found.owner}/${found.name}`;
|
const full = `${found.owner}/${found.name}`;
|
||||||
const pull = await this.getPullRequest(found.owner, found.name, found.number);
|
const pull = await this.getPullRequest(found.owner, found.name, found.number);
|
||||||
if (pull.state !== "open" || !pull.head_sha) continue;
|
if (pull.state !== "open" || !pull.head_sha) continue;
|
||||||
if (!rules.has(full)) rules.set(full, await this.branchProtections(found.owner, found.name).catch(() => undefined));
|
// The base's protection as the forge applies it to that branch (appliedProtection): readable without administration.
|
||||||
const rule = rules.get(full)?.find((p) => (p.rule_name ?? p.branch_name) === (pull.base || "main"));
|
const base = `${full}:${pull.base || "main"}`;
|
||||||
|
if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined));
|
||||||
|
const rule = rules.get(base);
|
||||||
const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
|
const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
|
||||||
const labels = found.labels ?? [];
|
const labels = found.labels ?? [];
|
||||||
// The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label.
|
// The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label.
|
||||||
@@ -673,16 +678,44 @@ export class GiteaClient {
|
|||||||
|
|
||||||
// ---- Branch protection ----
|
// ---- Branch protection ----
|
||||||
|
|
||||||
/** A repository's branch protection rules. */
|
/** A repository's branch protection rules. The forge answers them to a repository's administrator alone, which the
|
||||||
|
* tools' account is not (novox/hq ADR 0365 §2): for it this is refused (403), and what a branch's protection asks
|
||||||
|
* is read with appliedProtection. */
|
||||||
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
||||||
return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? [];
|
return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? [];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The rule named for a branch, null when it has none. */
|
/** The rule named for a branch, null when it has none — from the rules' list, so a repository administrator's. */
|
||||||
async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> {
|
async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> {
|
||||||
return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null;
|
return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The protection that covers an existing branch, as the forge applies it, or null when none does. Read from the
|
||||||
|
* branch itself, which anyone who reads the code may read — the rules' own list is a repository administrator's
|
||||||
|
* (novox/hq ADR 0365 §2) — and as the forge matches it, a glob rule included (its name is the rule's). The
|
||||||
|
* branch's `user_can_push` stands for `enable_push`: whether this account may push there. A branch that does not
|
||||||
|
* exist is null: nothing covers it yet, and the forge judges a push that makes it.
|
||||||
|
*/
|
||||||
|
async appliedProtection(owner: string, repo: string, branch: string): Promise<BranchProtection | null> {
|
||||||
|
let b: any;
|
||||||
|
try {
|
||||||
|
b = await this.request<any>(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`);
|
||||||
|
} catch (err) {
|
||||||
|
if (/: 404 /.test(String((err as Error)?.message))) return null;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
if (!b?.protected) return null;
|
||||||
|
return {
|
||||||
|
rule_name: String(b.effective_branch_protection_name || branch),
|
||||||
|
branch_name: branch,
|
||||||
|
enable_push: Boolean(b.user_can_push),
|
||||||
|
enable_status_check: Boolean(b.enable_status_check),
|
||||||
|
status_check_contexts: Array.isArray(b.status_check_contexts) ? b.status_check_contexts.map(String) : [],
|
||||||
|
required_approvals: Number(b.required_approvals ?? 0),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
|
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
|
||||||
* branch is edited — its required statuses replaced by these, everything else it says kept unless
|
* branch is edited — its required statuses replaced by these, everything else it says kept unless
|
||||||
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
|
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
|
||||||
@@ -732,6 +765,27 @@ export class GiteaClient {
|
|||||||
return this.requestText(`${repoPath(owner, repo)}/pulls/${index}.diff`);
|
return this.requestText(`${repoPath(owner, repo)}/pulls/${index}.diff`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** **Whether a pull request carries an approving review at a head** (novox/hq ADR 0382 §6): a review of the forge that
|
||||||
|
* approves, is not dismissed and was made at that very head; or the mesh's ready mark naming that head (issue 722), which is
|
||||||
|
* how an agent's approval is read where the forge refuses an agent's approve. Reads only. */
|
||||||
|
async approvedAt(owner: string, repo: string, index: number, head: string, authors?: string[]): Promise<boolean> {
|
||||||
|
indexOf(index);
|
||||||
|
const want = head.toLowerCase();
|
||||||
|
for (let page = 1; page <= 20; page++) {
|
||||||
|
const raw = await this.request<any[]>(`${repoPath(owner, repo)}/pulls/${index}/reviews?page=${page}&limit=50`);
|
||||||
|
for (const r of raw ?? []) {
|
||||||
|
if (String(r?.state ?? "").toUpperCase() === "APPROVED" && !r?.dismissed && !r?.stale &&
|
||||||
|
String(r?.commit_id ?? "").toLowerCase() === want) return true;
|
||||||
|
}
|
||||||
|
if (!raw || raw.length < 50) break;
|
||||||
|
}
|
||||||
|
const labels = await this.request<any[]>(`${repoPath(owner, repo)}/issues/${index}/labels`);
|
||||||
|
const label = (labels ?? []).find((l) => l?.name === READY_LABEL);
|
||||||
|
if (!label) return false;
|
||||||
|
const ready = readyMarkOf(Number(label.id ?? 0), await this.timeline(owner, repo, index), authors ?? readyAuthors());
|
||||||
|
return !!ready && ready.head.toLowerCase() === want;
|
||||||
|
}
|
||||||
|
|
||||||
/** **Every event of an issue's or pull request's timeline**, oldest first, page by page (novox/hq issue 722): a long
|
/** **Every event of an issue's or pull request's timeline**, oldest first, page by page (novox/hq issue 722): a long
|
||||||
* thread hides nothing. Bounded at 100 pages of 50; a longer one is said, and its first 5000 events read. */
|
* thread hides nothing. Bounded at 100 pages of 50; a longer one is said, and its first 5000 events read. */
|
||||||
async timeline(owner: string, repo: string, index: number): Promise<TimelineEvent[]> {
|
async timeline(owner: string, repo: string, index: number): Promise<TimelineEvent[]> {
|
||||||
@@ -925,6 +979,15 @@ export class GiteaAdmin {
|
|||||||
throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`);
|
throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A repository's branch protection rules, read with the administrator account — the forge answers them to a
|
||||||
|
* repository's administrator alone, which the tools' account is not (novox/hq ADR 0365 §2, ADR 0375). Only a read. */
|
||||||
|
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
|
||||||
|
const path = `${repoPath(owner, repo)}/branch_protections`;
|
||||||
|
const res = await this.request(path);
|
||||||
|
if (res.status !== 200) GiteaAdmin.fail(path, res);
|
||||||
|
return Array.isArray(res.body) ? res.body : [];
|
||||||
|
}
|
||||||
|
|
||||||
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
|
||||||
* create won the race) are all success. */
|
* create won the race) are all success. */
|
||||||
async ensureOrg(name: string): Promise<void> {
|
async ensureOrg(name: string): Promise<void> {
|
||||||
@@ -948,12 +1011,16 @@ export class GiteaAdmin {
|
|||||||
// unit the team does not have — so code read must be said here: without it gitea answers a
|
// unit the team does not have — so code read must be said here: without it gitea answers a
|
||||||
// member's clone of a private repository with "not found", which is how the builder's first
|
// member's clone of a private repository with "not found", which is how the builder's first
|
||||||
// credentialed clone failed against a team that named only packages.
|
// credentialed clone failed against a team that named only packages.
|
||||||
const units = {
|
return this.ensureTeamWith(org, team, {
|
||||||
permission: "read",
|
permission: "read",
|
||||||
units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" },
|
units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" },
|
||||||
includes_all_repositories: true,
|
includes_all_repositories: true,
|
||||||
can_create_org_repo: false,
|
can_create_org_repo: false,
|
||||||
};
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Ensure a team exists with exactly these units, and return its id (ensureTeam; the tools' team, tools-account.ts). */
|
||||||
|
async ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number> {
|
||||||
const found = await this.findTeam(org, team);
|
const found = await this.findTeam(org, team);
|
||||||
if (found !== null) {
|
if (found !== null) {
|
||||||
const patch = await this.request(`/teams/${found}`, {
|
const patch = await this.request(`/teams/${found}`, {
|
||||||
@@ -988,7 +1055,7 @@ export class GiteaAdmin {
|
|||||||
* The edit path is taken only when the user actually exists. A 422 from the create is also what
|
* The edit path is taken only when the user actually exists. A 422 from the create is also what
|
||||||
* a plain validation failure returns, and reading it as "already there" made the follow-up edit
|
* a plain validation failure returns, and reading it as "already there" made the follow-up edit
|
||||||
* 404 — burying the create's own message, which is the one that says what is actually wrong. */
|
* 404 — burying the create's own message, which is the one that says what is actually wrong. */
|
||||||
async ensureUser(username: string, password: string, email: string): Promise<void> {
|
async ensureUser(username: string, password: string, email: string, plain = false): Promise<void> {
|
||||||
const res = await this.request("/admin/users", {
|
const res = await this.request("/admin/users", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
body: JSON.stringify({ username, email, password, must_change_password: false }),
|
body: JSON.stringify({ username, email, password, must_change_password: false }),
|
||||||
@@ -1002,7 +1069,9 @@ export class GiteaAdmin {
|
|||||||
// login_name is required by the admin edit endpoint; for a local user it is the username.
|
// login_name is required by the admin edit endpoint; for a local user it is the username.
|
||||||
// active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
|
// active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
|
||||||
// password, so the provisioner's check reports it lost; applying again must undo both.
|
// password, so the provisioner's check reports it lost; applying again must undo both.
|
||||||
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }),
|
// plain: never a site administrator — one made so by hand is made a plain account again (novox/hq ADR 0365).
|
||||||
|
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false,
|
||||||
|
...(plain ? { admin: false } : {}) }),
|
||||||
});
|
});
|
||||||
if (patch.status === 200) return;
|
if (patch.status === 200) return;
|
||||||
GiteaAdmin.fail(`/admin/users/${username}`, patch);
|
GiteaAdmin.fail(`/admin/users/${username}`, patch);
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module
|
||||||
|
// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge
|
||||||
|
// refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the
|
||||||
|
// protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea/internal/forgehttp"
|
||||||
|
)
|
||||||
|
|
||||||
|
const toolsToken = "tools-token-never-shown"
|
||||||
|
|
||||||
|
// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are
|
||||||
|
// refused as to a non-administrator, and the branches named protected say so as the forge does.
|
||||||
|
func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
file := filepath.Join(dir, "token")
|
||||||
|
if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var seen []string
|
||||||
|
const repo = "/api/v1/repos/acme/app"
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
seen = append(seen, r.Method+" "+r.URL.Path)
|
||||||
|
if _, _, basic := r.BasicAuth(); basic || r.Header.Get("Authorization") != "token "+toolsToken {
|
||||||
|
http.Error(w, "unauthorized", 401)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.URL.Path == repo+"/branch_protections" {
|
||||||
|
http.Error(w, `{"message":"token does not have the administration of this repository"}`, 403)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if name := strings.TrimPrefix(r.URL.Path, repo+"/branches/"); r.Method == "GET" && name != r.URL.Path {
|
||||||
|
if rule, ok := protected[name]; ok {
|
||||||
|
_, _ = w.Write([]byte(`{"name":"` + name + `","commit":{"id":"c0"},"protected":true,"effective_branch_protection_name":"` + rule + `"}`))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.SetBasicAuth("mesh-admin", f.password) // the fake's own door, past the token checked above
|
||||||
|
f.handle(w, r)
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools")
|
||||||
|
got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.writes) != 1 || !got.Created {
|
||||||
|
t.Fatalf("%d writes, %+v", len(f.writes), got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefaultBranch(t *testing.T) {
|
||||||
|
for branch, rule := range map[string]string{"release/1.0": "release/*", "fix/1-old": "fix/1-old", "main": ""} {
|
||||||
|
f := newFake(t)
|
||||||
|
protected := map[string]string{}
|
||||||
|
if rule != "" {
|
||||||
|
protected[branch] = rule
|
||||||
|
}
|
||||||
|
forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools")
|
||||||
|
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch,
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") {
|
||||||
|
t.Errorf("%s: %v", branch, err)
|
||||||
|
}
|
||||||
|
if rule != "" && !strings.Contains(err.Error(), rule) {
|
||||||
|
t.Errorf("%s: the rule %s is not named: %v", branch, rule, err)
|
||||||
|
}
|
||||||
|
if len(f.writes)+len(f.patches) != 0 {
|
||||||
|
t.Errorf("%s: something was written", branch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
// What the module kept before ADR 0365: the admin's token, its scopes line naming no account.
|
||||||
|
forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository")
|
||||||
|
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
|
||||||
|
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
if len(*seen) != 0 {
|
||||||
|
t.Fatalf("the forge was asked %v", *seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) {
|
||||||
|
t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000")
|
||||||
|
t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token")
|
||||||
|
t.Setenv("MESH_GITEA_TOOLS_USER", "")
|
||||||
|
t.Setenv("MESH_GITEA_ADMIN_USER", "")
|
||||||
|
t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "")
|
||||||
|
f, err := ForgeFromEnv()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" {
|
||||||
|
t.Fatalf("%+v", f.Client)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,8 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea/internal/forgehttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull
|
// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull
|
||||||
@@ -33,6 +35,11 @@ type CommitAsk struct {
|
|||||||
Message string
|
Message string
|
||||||
Files []FileChange
|
Files []FileChange
|
||||||
Patch string // a unified diff, as git diff writes it, instead of Files
|
Patch string // a unified diff, as git diff writes it, instead of Files
|
||||||
|
// PatchFile is the name of a file in the spool directory holding the patch (novox/hq issue 852), with the
|
||||||
|
// SHA-256 its caller says it has; commitAsk reads it into Patch.
|
||||||
|
PatchFile string
|
||||||
|
PatchSHA256 string
|
||||||
|
patchFileGiven bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Committed is what gitea_commit made.
|
// Committed is what gitea_commit made.
|
||||||
@@ -67,6 +74,11 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"),
|
a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"),
|
||||||
Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))}
|
Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))}
|
||||||
a.Patch = raw(args, "patch")
|
a.Patch = raw(args, "patch")
|
||||||
|
if v, ok := args["patch_file"]; ok && v != nil {
|
||||||
|
a.patchFileGiven = true
|
||||||
|
a.PatchFile = raw(args, "patch_file")
|
||||||
|
}
|
||||||
|
a.PatchSHA256 = strings.ToLower(strings.TrimSpace(raw(args, "patch_sha256")))
|
||||||
if a.Branch == "" {
|
if a.Branch == "" {
|
||||||
return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent")
|
return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent")
|
||||||
}
|
}
|
||||||
@@ -109,11 +121,23 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
a.Files = append(a.Files, fc)
|
a.Files = append(a.Files, fc)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sources := 0
|
||||||
|
for _, g := range []bool{len(a.Files) > 0, strings.TrimSpace(a.Patch) != "", a.patchFileGiven} {
|
||||||
|
if g {
|
||||||
|
sources++
|
||||||
|
}
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case len(a.Files) == 0 && strings.TrimSpace(a.Patch) == "":
|
case sources == 0:
|
||||||
return a, refuse("give files (each {path, content | content_base64 | delete: true}) or patch (a unified diff, as git diff writes it). Nothing was sent")
|
return a, refuse("give files (each {path, content | content_base64 | delete: true}), patch (a unified diff, as git diff writes it) or patch_file (the name of a patch in the spool directory). Nothing was sent")
|
||||||
case len(a.Files) > 0 && strings.TrimSpace(a.Patch) != "":
|
case sources > 1:
|
||||||
return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent")
|
return a, refuse("give files, patch or patch_file, not more than one: one commit carries one of them. Nothing was sent")
|
||||||
|
case a.PatchSHA256 != "" && !a.patchFileGiven:
|
||||||
|
return a, refuse("patch_sha256 goes with patch_file only. Nothing was sent")
|
||||||
|
case a.patchFileGiven:
|
||||||
|
if err := checkPatchFileAsk(a); err != nil {
|
||||||
|
return a, err
|
||||||
|
}
|
||||||
case len(a.Files) > maxFiles:
|
case len(a.Files) > maxFiles:
|
||||||
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
|
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
|
||||||
case len(a.Patch) > maxPatchBytes:
|
case len(a.Patch) > maxPatchBytes:
|
||||||
@@ -132,13 +156,13 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
|
|||||||
return a, refuse("%s is named twice in files. Nothing was sent", fc.Path)
|
return a, refuse("%s is named twice in files. Nothing was sent", fc.Path)
|
||||||
}
|
}
|
||||||
seen[fc.Path] = true
|
seen[fc.Path] = true
|
||||||
given := 0
|
have := 0
|
||||||
for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} {
|
for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} {
|
||||||
if g {
|
if g {
|
||||||
given++
|
have++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if given != 1 {
|
if have != 1 {
|
||||||
return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path)
|
return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path)
|
||||||
}
|
}
|
||||||
if fc.Content != nil && !utf8.ValidString(*fc.Content) {
|
if fc.Content != nil && !utf8.ValidString(*fc.Content) {
|
||||||
@@ -211,7 +235,32 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
|
|||||||
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
|
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
|
||||||
var rules []Protection
|
var rules []Protection
|
||||||
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
|
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
|
||||||
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
|
var status *forgehttp.StatusError
|
||||||
|
if !errors.As(err, &status) || status.Status != http.StatusForbidden {
|
||||||
|
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
|
||||||
|
}
|
||||||
|
// The rules are a repository administrator's to read, and the tools' account is none (novox/hq ADR 0365
|
||||||
|
// §2). What it may read is the protection the forge applies to an existing branch: one protected is
|
||||||
|
// refused here. A branch not made yet is covered by no rule this account can see, and the forge itself
|
||||||
|
// judges the commit that makes it against every rule, refusing one a rule guards.
|
||||||
|
if err := targetRefused(a.Branch, r.DefaultBranch, nil); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b, err := f.branch(ctx, repo, a.Branch)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("whether a protection rule guards %s of %s/%s could not be read, so nothing was sent: %w", a.Branch, a.Owner, a.Repo, err)
|
||||||
|
}
|
||||||
|
if b != nil && b.Protected {
|
||||||
|
rule := b.Rule
|
||||||
|
if rule == "" {
|
||||||
|
rule = a.Branch
|
||||||
|
}
|
||||||
|
rules = []Protection{{RuleName: rule}}
|
||||||
|
if !ruleCovers(rule, a.Branch) {
|
||||||
|
return nil, refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+
|
||||||
|
"branch of its own. Nothing was sent", a.Branch, rule)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
|
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import (
|
|||||||
"gitea/internal/forgehttp"
|
"gitea/internal/forgehttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account
|
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq
|
||||||
// (internal/forgehttp, shared with the npm-registry bundle beside it).
|
// ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it).
|
||||||
|
|
||||||
// Forge is the forge's HTTP interface.
|
// Forge is the forge's HTTP interface.
|
||||||
type Forge struct {
|
type Forge struct {
|
||||||
@@ -91,12 +91,15 @@ func (p Protection) name() string {
|
|||||||
return p.BranchName
|
return p.BranchName
|
||||||
}
|
}
|
||||||
|
|
||||||
// Branch is one branch with the commit it points at.
|
// Branch is one branch with the commit it points at, and the protection the forge applies to it: what an account
|
||||||
|
// that may not read the rules themselves (the tools' account, novox/hq ADR 0365) knows of them.
|
||||||
type Branch struct {
|
type Branch struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Commit struct {
|
Commit struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
} `json:"commit"`
|
} `json:"commit"`
|
||||||
|
Protected bool `json:"protected"`
|
||||||
|
Rule string `json:"effective_branch_protection_name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// branch is the named branch, nil when the forge has none of that name.
|
// branch is the named branch, nil when the forge has none of that name.
|
||||||
|
|||||||
@@ -619,11 +619,17 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) {
|
|||||||
if len(a.Loads) != 1 || a.Loads[0] != a.Binary {
|
if len(a.Loads) != 1 || a.Loads[0] != a.Binary {
|
||||||
t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary)
|
t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} {
|
// The tools' own account, never the admin login (novox/hq ADR 0365 §2).
|
||||||
|
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} {
|
||||||
if a.Env[k] == "" {
|
if a.Env[k] == "" {
|
||||||
t.Errorf("the bundle is not given %s", k)
|
t.Errorf("the bundle is not given %s", k)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} {
|
||||||
|
if a.Env[k] != "" {
|
||||||
|
t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
|
|||||||
@@ -55,7 +55,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
|
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
|
||||||
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
|
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
|
||||||
"the parent the forge made it on, read back from the branch, and says when the branch moved under " +
|
"the parent the forge made it on, read back from the branch, and says when the branch moved under " +
|
||||||
"it. Replaces git commit and git " +
|
"it. A patch too large to type goes as patch_file with patch_sha256: the name of a file in the " +
|
||||||
|
"spool directory (read once, applied as patch is, removed afterwards, never read from anywhere " +
|
||||||
|
"else). Replaces git commit and git " +
|
||||||
"push of a feature branch. (a)",
|
"push of a feature branch. (a)",
|
||||||
Input: map[string]any{
|
Input: map[string]any{
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -79,7 +81,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
"required": []string{"path"},
|
"required": []string{"path"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this or files"),
|
"patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this, patch_file or files"),
|
||||||
|
"patch_file": str("for a patch too large to type: the NAME (32 to 64 characters of 0-9a-f, chosen at random by the caller, no path) of a file the caller wrote into the forge patch spool directory (the module's forge-patches directory); needs patch_sha256; give this, patch or files"),
|
||||||
|
"patch_sha256": str("with patch_file: the file's SHA-256, in hex; the commit is refused, and nothing is sent, when the file's differs"),
|
||||||
},
|
},
|
||||||
"required": []string{"owner", "repo", "branch", "message"},
|
"required": []string{"owner", "repo", "branch", "message"},
|
||||||
},
|
},
|
||||||
@@ -88,7 +92,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return commit(ctx, f, a)
|
return commitAsk(ctx, f, os.Getenv(patchDirEnv), a)
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A patch too large to type into a call is handed over as a file (novox/hq issue 852). The caller writes it into
|
||||||
|
// one spool directory the module declares (mode 0733: anyone writes, nobody lists) and names it in patch_file,
|
||||||
|
// with the file's SHA-256. The tool runs as the machine's account, so it reads nothing but that directory: the
|
||||||
|
// name is 32 to 64 characters of [0-9a-f], so it holds no separator and no dot; the file must be a regular file
|
||||||
|
// (never a symlink); and its hash must be the caller's, because without the sticky bit another writer could have
|
||||||
|
// replaced it. The file is removed once taken, whether the commit was made or not.
|
||||||
|
|
||||||
|
const patchDirEnv = "MESH_GITEA_PATCH_DIR"
|
||||||
|
|
||||||
|
var (
|
||||||
|
patchFileName = regexp.MustCompile(`^[0-9a-f]{32,64}$`)
|
||||||
|
patchSHA = regexp.MustCompile(`^[0-9a-f]{64}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// checkPatchFileAsk judges the arguments of a patch_file ask, before any file is looked at.
|
||||||
|
func checkPatchFileAsk(a CommitAsk) error {
|
||||||
|
if !patchFileName.MatchString(a.PatchFile) {
|
||||||
|
return refuse("patch_file %q is not a name: 32 to 64 characters of 0-9a-f, no path (the file's name in the spool directory, chosen at random). Nothing was sent", a.PatchFile)
|
||||||
|
}
|
||||||
|
if !patchSHA.MatchString(a.PatchSHA256) {
|
||||||
|
return refuse("patch_file needs patch_sha256: the file's SHA-256, 64 hex characters. Nothing was sent")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// commitAsk makes the commit the ask names; a patch_file is first read from the spool directory into the patch
|
||||||
|
// and removed afterwards.
|
||||||
|
func commitAsk(ctx context.Context, f *Forge, spool string, a CommitAsk) (*Committed, error) {
|
||||||
|
if !a.patchFileGiven {
|
||||||
|
return commit(ctx, f, a)
|
||||||
|
}
|
||||||
|
if err := checkPatchFileAsk(a); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if spool == "" {
|
||||||
|
return nil, refuse("patch_file is not available: %s is not set, so there is no spool directory. Nothing was sent", patchDirEnv)
|
||||||
|
}
|
||||||
|
path := filepath.Join(spool, a.PatchFile)
|
||||||
|
defer func() {
|
||||||
|
// Whatever happened, the file is not left: removing a symlink removes the link, never its target.
|
||||||
|
if st, err := os.Lstat(path); err == nil && !st.IsDir() {
|
||||||
|
_ = os.Remove(path)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
body, err := readSpooled(path, a.PatchFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
got := sha256.Sum256(body)
|
||||||
|
if hex.EncodeToString(got[:]) != a.PatchSHA256 {
|
||||||
|
return nil, refuse("the file %s is not the patch the caller hashed: its SHA-256 is %s, patch_sha256 says %s. Nothing was sent",
|
||||||
|
a.PatchFile, hex.EncodeToString(got[:]), a.PatchSHA256)
|
||||||
|
}
|
||||||
|
a.Patch = string(body)
|
||||||
|
return commit(ctx, f, a)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSpooled reads a regular file of at most maxPatchBytes, never through a symlink.
|
||||||
|
func readSpooled(path, name string) ([]byte, error) {
|
||||||
|
st, err := os.Lstat(path)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, os.ErrNotExist):
|
||||||
|
return nil, refuse("patch_file %s: no such file in the spool directory. Nothing was sent", name)
|
||||||
|
case err != nil:
|
||||||
|
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
|
||||||
|
case !st.Mode().IsRegular():
|
||||||
|
return nil, refuse("patch_file %s is not a regular file (a symlink or a directory is never followed). Nothing was sent", name)
|
||||||
|
case st.Size() > maxPatchBytes:
|
||||||
|
return nil, refuse("patch_file %s is %s, more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(int(st.Size())), mib(maxPatchBytes))
|
||||||
|
}
|
||||||
|
fh, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, refuse("patch_file %s cannot be opened: %v. Nothing was sent", name, err)
|
||||||
|
}
|
||||||
|
defer fh.Close()
|
||||||
|
fst, err := fh.Stat()
|
||||||
|
if err != nil || !fst.Mode().IsRegular() {
|
||||||
|
return nil, refuse("patch_file %s is not a regular file. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
// A hard link to a file elsewhere is no symlink, but it is not the caller's own file either.
|
||||||
|
if sys, ok := fst.Sys().(*syscall.Stat_t); !ok || sys.Nlink > 1 {
|
||||||
|
return nil, refuse("patch_file %s has more than one hard link: it must be a file of its own. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(fh, maxPatchBytes+1))
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
|
||||||
|
case len(body) > maxPatchBytes:
|
||||||
|
return nil, refuse("patch_file %s is more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(maxPatchBytes))
|
||||||
|
case strings.TrimSpace(string(body)) == "":
|
||||||
|
return nil, refuse("patch_file %s is empty. Nothing was sent", name)
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 852: a patch too large to type into a call is handed over as a file in one spool directory.
|
||||||
|
|
||||||
|
const spoolName = "0123456789abcdef0123456789abcdef"
|
||||||
|
const spoolPatch = "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n"
|
||||||
|
|
||||||
|
func sum(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||||
|
|
||||||
|
func spoolWith(t *testing.T, name, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
dir := t.TempDir()
|
||||||
|
if name != "" {
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dir
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileAsk(t *testing.T, name, sha string) CommitAsk {
|
||||||
|
t.Helper()
|
||||||
|
return ask(t, map[string]any{"branch": "fix/9", "patch_file": name, "patch_sha256": sha})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileIsAppliedAsThePatchWouldBeAndTheFileIsGone(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := spoolWith(t, spoolName, spoolPatch)
|
||||||
|
got, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, spoolName, sum(spoolPatch)))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(f.patches) != 1 || f.patches[0]["content"] != spoolPatch || f.patches[0]["new_branch"] != "fix/9" || got.Commit != "p1" {
|
||||||
|
t.Fatalf("%v %+v", f.patches, got)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("the file is still there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileIsRemovedWhenTheForgeRefusesIt(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := spoolWith(t, spoolName, spoolPatch)
|
||||||
|
// the default branch is refused after the file was read
|
||||||
|
a := ask(t, map[string]any{"branch": "main", "patch_file": spoolName, "patch_sha256": sum(spoolPatch)})
|
||||||
|
if _, err := commitAsk(context.Background(), f.serve(), dir, a); err == nil {
|
||||||
|
t.Fatal("taken")
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("the file is still there: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileThatCannotBeTakenIsRefusedBeforeAnythingIsSent(t *testing.T) {
|
||||||
|
outside := filepath.Join(t.TempDir(), "secret")
|
||||||
|
if err := os.WriteFile(outside, []byte(spoolPatch), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
type tc struct {
|
||||||
|
name string
|
||||||
|
setup func(dir string)
|
||||||
|
file string
|
||||||
|
sha string
|
||||||
|
want string
|
||||||
|
}
|
||||||
|
for _, c := range []tc{
|
||||||
|
{"wrong sha", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), []byte(spoolPatch), 0o600) }, spoolName, sum("other"), "sha256"},
|
||||||
|
{"not found", func(string) {}, spoolName, sum(spoolPatch), "no such file"},
|
||||||
|
{"symlink", func(d string) { _ = os.Symlink(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "regular file"},
|
||||||
|
{"hard link", func(d string) { _ = os.Link(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "hard link"},
|
||||||
|
{"directory", func(d string) { _ = os.Mkdir(filepath.Join(d, spoolName), 0o700) }, spoolName, sum(spoolPatch), "regular file"},
|
||||||
|
{"too large", func(d string) {
|
||||||
|
_ = os.WriteFile(filepath.Join(d, spoolName), []byte(strings.Repeat("x", maxPatchBytes+1)), 0o600)
|
||||||
|
}, spoolName, sum(spoolPatch), "MiB"},
|
||||||
|
{"empty", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), nil, 0o600) }, spoolName, sum(""), "empty"},
|
||||||
|
} {
|
||||||
|
f := newFake(t)
|
||||||
|
dir := t.TempDir()
|
||||||
|
c.setup(dir)
|
||||||
|
_, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, c.file, c.sha))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Errorf("%s: %v", c.name, err)
|
||||||
|
}
|
||||||
|
if len(f.patches)+len(f.writes) != 0 {
|
||||||
|
t.Errorf("%s: something was sent", c.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b, err := os.ReadFile(outside); err != nil || string(b) != spoolPatch {
|
||||||
|
t.Errorf("the file outside the spool was touched: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileAskThatCannotBeReadIsRefused(t *testing.T) {
|
||||||
|
ok := sum(spoolPatch)
|
||||||
|
for name, args := range map[string]map[string]any{
|
||||||
|
"no sha": {"patch_file": spoolName},
|
||||||
|
"sha not hex": {"patch_file": spoolName, "patch_sha256": strings.Repeat("z", 64)},
|
||||||
|
"sha too short": {"patch_file": spoolName, "patch_sha256": ok[:63]},
|
||||||
|
"empty name": {"patch_file": "", "patch_sha256": ok, "files": filesArg()},
|
||||||
|
"dotdot": {"patch_file": "../" + spoolName, "patch_sha256": ok},
|
||||||
|
"slash": {"patch_file": "a/" + spoolName, "patch_sha256": ok},
|
||||||
|
"absolute": {"patch_file": "/etc/passwd", "patch_sha256": ok},
|
||||||
|
"too short": {"patch_file": strings.Repeat("a", 31), "patch_sha256": ok},
|
||||||
|
"too long": {"patch_file": strings.Repeat("a", 65), "patch_sha256": ok},
|
||||||
|
"upper case": {"patch_file": strings.ToUpper(spoolName), "patch_sha256": ok},
|
||||||
|
"not hex": {"patch_file": strings.Repeat("g", 32), "patch_sha256": ok},
|
||||||
|
"with patch": {"patch_file": spoolName, "patch_sha256": ok, "patch": "diff"},
|
||||||
|
"with files": {"patch_file": spoolName, "patch_sha256": ok, "files": filesArg(map[string]any{"path": "x", "content": "x"})},
|
||||||
|
"sha without it": {"patch": "diff", "patch_sha256": ok},
|
||||||
|
} {
|
||||||
|
full := map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b"}
|
||||||
|
for k, v := range args {
|
||||||
|
full[k] = v
|
||||||
|
}
|
||||||
|
_, err := readCommitAsk(full)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Errorf("%s: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 64 characters is taken; so is an upper-case sha.
|
||||||
|
for _, n := range []string{spoolName, strings.Repeat("ab", 32)} {
|
||||||
|
if _, err := readCommitAsk(map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b",
|
||||||
|
"patch_file": n, "patch_sha256": strings.ToUpper(sum(spoolPatch))}); err != nil {
|
||||||
|
t.Errorf("%s: %v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPatchFileWithoutASpoolDirectoryIsRefused(t *testing.T) {
|
||||||
|
f := newFake(t)
|
||||||
|
_, err := commitAsk(context.Background(), f.serve(), "", fileAsk(t, spoolName, sum(spoolPatch)))
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "MESH_GITEA_PATCH_DIR") || !strings.Contains(err.Error(), "Nothing was sent") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheManifestDeclaresTheSpoolDirectoryAndGivesItToTheBundle(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m struct {
|
||||||
|
Resources []struct {
|
||||||
|
ID, Type, Path, Mode, Owner string
|
||||||
|
} `json:"resources"`
|
||||||
|
Build struct {
|
||||||
|
Artifacts []struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
Env map[string]string `json:"env"`
|
||||||
|
} `json:"artifacts"`
|
||||||
|
} `json:"build"`
|
||||||
|
}
|
||||||
|
if err := jsonInto(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
found := false
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r.ID == "forge-patches" {
|
||||||
|
found = true
|
||||||
|
if r.Type != "directory" || r.Mode != "0733" || r.Owner != "${machine:account}" || r.Path == "" {
|
||||||
|
t.Errorf("%+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Error("no forge-patches directory")
|
||||||
|
}
|
||||||
|
for _, a := range m.Build.Artifacts {
|
||||||
|
if a.From == "cmd/forge" && a.Env["MESH_GITEA_PATCH_DIR"] != "${dir:forge-patches}" {
|
||||||
|
t.Errorf("env %v", a.Env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+62
-29
@@ -42,9 +42,10 @@ import { commitStatusTool } from "./delivery.js";
|
|||||||
import { mergeTool } from "./merge.js";
|
import { mergeTool } from "./merge.js";
|
||||||
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
|
||||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
|
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
|
||||||
type RepoCheckFacts, reposToLook } from "./pulls.js";
|
repoCheckFacts, reposToLook } from "./pulls.js";
|
||||||
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
|
||||||
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
|
||||||
|
import { keepSiblings, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
|
||||||
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
|
||||||
|
|
||||||
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
|
||||||
@@ -136,6 +137,7 @@ async function pollMerged(client: GiteaClient): Promise<void> {
|
|||||||
await emit("pull.closed", { owner: repo.owner, repo: repo.name, number: pull.number, title: pull.title,
|
await emit("pull.closed", { owner: repo.owner, repo: repo.name, number: pull.number, title: pull.title,
|
||||||
head: pull.head, head_sha: pull.head_sha, base: pull.base, html_url: pull.html_url });
|
head: pull.head, head_sha: pull.head_sha, base: pull.base, html_url: pull.html_url });
|
||||||
console.log(`[gitea] announced ${repo.full_name}#${pull.number} closed unmerged`);
|
console.log(`[gitea] announced ${repo.full_name}#${pull.number} closed unmerged`);
|
||||||
|
await siblingMoves(client, repo.full_name, pull.number);
|
||||||
}
|
}
|
||||||
announced.add(closedKey);
|
announced.add(closedKey);
|
||||||
changed = true;
|
changed = true;
|
||||||
@@ -218,6 +220,8 @@ async function announceMerge(client: GiteaClient, repo: RepoOf, pull: PullOf): P
|
|||||||
// Said, because a trigger that fires silently is indistinguishable from one that did not
|
// Said, because a trigger that fires silently is indistinguishable from one that did not
|
||||||
// fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told.
|
// fire (novox/hq 04-ISSUES/131) — this line is how an operator knows the mesh was told.
|
||||||
console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`);
|
console.log(`[gitea] announced merge ${repo.full_name}#${pull.number} (${pull.merge_commit_sha.slice(0, 8)}) into ${pull.base}`);
|
||||||
|
// Its own merge of the head that was read is no sibling's move; any other is (ADR 0382 §5).
|
||||||
|
await siblingMoves(client, repo.full_name, pull.number);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
announced.delete(pull.merge_commit_sha);
|
announced.delete(pull.merge_commit_sha);
|
||||||
throw err;
|
throw err;
|
||||||
@@ -264,6 +268,8 @@ async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Pr
|
|||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
keepHeads();
|
keepHeads();
|
||||||
|
// A sibling's new head puts out of date every verdict that read the sibling at another head (ADR 0382 §5).
|
||||||
|
await siblingMoves(client, repo.full_name, pull.number);
|
||||||
// Said, so an operator knows the mesh was asked to check it.
|
// Said, so an operator knows the mesh was asked to check it.
|
||||||
console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`);
|
console.log(`[gitea] announced ${key} at ${sha.slice(0, 8)} to be checked before it merges`);
|
||||||
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
|
// Pending until the verdict comes, so the pull request says a check is running rather than nothing — and
|
||||||
@@ -341,7 +347,22 @@ function keepTrunks(): void {
|
|||||||
|
|
||||||
/** Whether a trunk of one repository moved, weighed with this module's forge and its one writer (trunk.ts). */
|
/** Whether a trunk of one repository moved, weighed with this module's forge and its one writer (trunk.ts). */
|
||||||
function trunksMoved(client: GiteaClient, owner: string, name: string): Promise<boolean> {
|
function trunksMoved(client: GiteaClient, owner: string, name: string): Promise<boolean> {
|
||||||
return weighTrunks(client, gates, trunks, owner, name);
|
return weighTrunks(client, gates, trunks, owner, name, () => siblingMoves(client, `${owner}/${name}`));
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The siblings each open pull request's latest verdict names** (novox/hq ADR 0382 §5, siblings.ts), by member, kept beside
|
||||||
|
// the other records. A sibling's new head, close unmerged, merge, or a move of its base puts the merge check of every
|
||||||
|
// member whose verdict names it back to pending, with the reason; nothing is run for it.
|
||||||
|
const siblingsRecord = siblingsFile(process.env.MESH_GITEA_STATE_DIR);
|
||||||
|
const siblingNamed = readSiblings(siblingsRecord);
|
||||||
|
|
||||||
|
/** A sibling moved, weighed (ADR 0382 §5): never fatal to what heard it. */
|
||||||
|
async function siblingMoves(client: GiteaClient, repository: string, number?: number): Promise<void> {
|
||||||
|
try {
|
||||||
|
if (await weighSiblings(client, gates, siblingNamed, repository, number)) keepSiblings(siblingsRecord, siblingNamed);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] whether a sibling of ${repository}${number ? "#" + number : ""} moved could not be weighed — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// **The verdict, set where the pull request shows it.** Every verdict is the head commit's status; one
|
// **The verdict, set where the pull request shows it.** Every verdict is the head commit's status; one
|
||||||
@@ -363,6 +384,17 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
|||||||
console.error("[gitea] a merge check's verdict named no repository, commit or verdict; ignored");
|
console.error("[gitea] a merge check's verdict named no repository, commit or verdict; ignored");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// The siblings this verdict read are kept for the member (ADR 0382 §5); a verdict that read none removes the record.
|
||||||
|
const siblings = siblingsOf(c.siblings);
|
||||||
|
if (c.number && noteVerdict(siblingNamed, `${c.owner}/${c.repo}#${c.number}`.toLowerCase(), c.commit, siblings)) {
|
||||||
|
keepSiblings(siblingsRecord, siblingNamed);
|
||||||
|
}
|
||||||
|
// A sibling that moved while the check ran is weighed once this verdict has landed, in the head's own turn after it.
|
||||||
|
if (siblings.length) {
|
||||||
|
for (const repository of new Set(siblings.map((s) => s.repository))) {
|
||||||
|
void Promise.resolve().then(() => siblingMoves(client, repository));
|
||||||
|
}
|
||||||
|
}
|
||||||
// Each status links to the pull request, where the delivery's view is kept (novox/hq ADR 0239).
|
// Each status links to the pull request, where the delivery's view is kept (novox/hq ADR 0239).
|
||||||
let target: string | undefined;
|
let target: string | undefined;
|
||||||
let base: string | undefined;
|
let base: string | undefined;
|
||||||
@@ -400,22 +432,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
|
|||||||
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
|
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
|
|
||||||
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
|
|
||||||
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
|
|
||||||
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
|
|
||||||
// required would let an untested repository merge.
|
|
||||||
async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
|
|
||||||
if (c["repo-check"]?.verdict !== "warning") return {};
|
|
||||||
const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
|
|
||||||
if (defined !== false) return { defined };
|
|
||||||
const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined);
|
|
||||||
if (!rules) return { defined };
|
|
||||||
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main"));
|
|
||||||
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
|
|
||||||
return { defined, required };
|
|
||||||
}
|
|
||||||
|
|
||||||
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
|
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
|
||||||
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
|
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
|
||||||
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
|
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
|
||||||
@@ -429,10 +445,16 @@ const gates = new GateWriter({
|
|||||||
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
|
||||||
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
|
||||||
// reach, nothing is served, as the tools' bundle serves nothing then.
|
// reach, nothing is served, as the tools' bundle serves nothing then.
|
||||||
registerModuleTools("gitea", () => (gitea ? [commitStatusTool({
|
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
|
||||||
|
* gitea_events says of them (webhook.ts). */
|
||||||
|
const pace = new Pace();
|
||||||
|
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
|
||||||
|
|
||||||
|
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
|
||||||
|
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
|
||||||
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
|
||||||
gates,
|
gates,
|
||||||
}), mergeTool(gitea, { gates })] : []));
|
}), mergeTool(gitea, { gates })] : [])]);
|
||||||
|
|
||||||
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
|
||||||
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
|
||||||
@@ -451,13 +473,15 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */
|
|
||||||
const pace = new Pace();
|
|
||||||
|
|
||||||
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
|
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
|
||||||
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
|
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
|
||||||
const b = event.body as { owner?: string; repo?: string; number?: number };
|
const b = event.body as { owner?: string; repo?: string; number?: number };
|
||||||
if (!b.owner || !b.repo || !b.number) return;
|
if (!b.owner || !b.repo || !b.number) return;
|
||||||
|
if (event.name === "forge.pull-request.closed") {
|
||||||
|
// Closed unmerged is a sibling's move; a merge is weighed with its announcement below (ADR 0382 §5).
|
||||||
|
if (!(event.body as { merged?: boolean }).merged) await siblingMoves(client, `${b.owner}/${b.repo}`, b.number);
|
||||||
|
return;
|
||||||
|
}
|
||||||
const head = ["forge.pull-request.opened", "forge.pull-request.synchronized", "forge.pull-request.reopened"].includes(event.name);
|
const head = ["forge.pull-request.opened", "forge.pull-request.synchronized", "forge.pull-request.reopened"].includes(event.name);
|
||||||
const merged = event.name === "forge.pull-request.merged";
|
const merged = event.name === "forge.pull-request.merged";
|
||||||
if (!head && !merged) return;
|
if (!head && !merged) return;
|
||||||
@@ -476,23 +500,32 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
|
|||||||
const url = process.env.MESH_GITEA_HOOK_URL;
|
const url = process.env.MESH_GITEA_HOOK_URL;
|
||||||
if (!stateDir || !url) {
|
if (!stateDir || !url) {
|
||||||
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
|
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
|
||||||
|
pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const secret = keptSecret(join(stateDir, "webhook.secret"));
|
const secret = keptSecret(join(stateDir, "webhook.secret"));
|
||||||
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
|
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
|
||||||
const server = receiver({ secret, port, heard: () => pace.heard(),
|
const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
|
||||||
|
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
|
||||||
|
refused: (reason, from) => pace.refused(reason, from),
|
||||||
publish: (event) => emit(event.name, event.body),
|
publish: (event) => emit(event.name, event.body),
|
||||||
act: (event) => actOn(client, event) });
|
act: (event) => actOn(client, event) });
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
server.once("error", reject);
|
server.once("error", reject);
|
||||||
server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve());
|
server.listen(port, host, () => resolve());
|
||||||
});
|
});
|
||||||
const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
|
const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
|
||||||
console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`);
|
pace.listening(host, port, hook.url, hook);
|
||||||
|
console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
|
||||||
|
`at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
|
||||||
|
`${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
|
||||||
|
`${hook.events.length} event families, signed`);
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`);
|
const why = err instanceof Error ? err.message : String(err);
|
||||||
|
pace.unheard(why);
|
||||||
|
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge,
|
// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge,
|
||||||
// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered
|
// cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539).
|
||||||
// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call
|
//
|
||||||
// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539).
|
// Two ways in, by what the manifest hands the bundle:
|
||||||
|
//
|
||||||
|
// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module
|
||||||
|
// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A
|
||||||
|
// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's —
|
||||||
|
// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints
|
||||||
|
// nothing; a token the forge rejects is renewed by the module's next call, and said here.
|
||||||
|
// - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR
|
||||||
|
// 0086), by basic authentication, when no token file is named.
|
||||||
|
//
|
||||||
|
// A credential is read when a call is made and never logged, answered or put in an error.
|
||||||
package forgehttp
|
package forgehttp
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -25,12 +35,31 @@ type Client struct {
|
|||||||
URL string // e.g. http://127.0.0.1:3000
|
URL string // e.g. http://127.0.0.1:3000
|
||||||
User string
|
User string
|
||||||
PasswordFile string
|
PasswordFile string
|
||||||
HTTP *http.Client // when nil, a client with Timeout
|
// TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login;
|
||||||
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero
|
// Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools).
|
||||||
|
TokenFile string
|
||||||
|
Account string
|
||||||
|
HTTP *http.Client // when nil, a client with Timeout
|
||||||
|
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset.
|
// ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts).
|
||||||
|
const ToolsAccount = "mesh-tools"
|
||||||
|
|
||||||
|
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle
|
||||||
|
// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login.
|
||||||
func FromEnv() (Client, []string) {
|
func FromEnv() (Client, []string) {
|
||||||
|
if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" {
|
||||||
|
c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file,
|
||||||
|
Account: os.Getenv("MESH_GITEA_TOOLS_USER")}
|
||||||
|
if c.Account == "" {
|
||||||
|
c.Account = ToolsAccount
|
||||||
|
}
|
||||||
|
if c.URL == "" {
|
||||||
|
return c, []string{"MESH_GITEA_URL"}
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
c := Client{
|
c := Client{
|
||||||
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
|
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
|
||||||
User: os.Getenv("MESH_GITEA_ADMIN_USER"),
|
User: os.Getenv("MESH_GITEA_ADMIN_USER"),
|
||||||
@@ -80,14 +109,45 @@ func (c *Client) password() (string, error) {
|
|||||||
return strings.TrimSpace(string(raw)), nil
|
return strings.TrimSpace(string(raw)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// token is the tools' account's token, refused unless its scopes line names that account.
|
||||||
|
func (c *Client) token() (string, error) {
|
||||||
|
raw, err := os.ReadFile(c.TokenFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err)
|
||||||
|
}
|
||||||
|
line, err := os.ReadFile(c.TokenFile + ".scopes")
|
||||||
|
if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) {
|
||||||
|
return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account)
|
||||||
|
}
|
||||||
|
token := strings.TrimSpace(string(raw))
|
||||||
|
if token == "" {
|
||||||
|
return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile)
|
||||||
|
}
|
||||||
|
return token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// authorize puts the bundle's credential on a request: the tools' token, or the admin login.
|
||||||
|
func (c *Client) authorize(req *http.Request) error {
|
||||||
|
if c.TokenFile != "" {
|
||||||
|
token, err := c.token()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "token "+token)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
password, err := c.password()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.SetBasicAuth(c.User, password)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit;
|
// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit;
|
||||||
// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it
|
// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it
|
||||||
// exactly: a cut body is refused, never read as if it were whole.
|
// exactly: a cut body is refused, never read as if it were whole.
|
||||||
func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
|
func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
|
||||||
password, err := c.password()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
var body io.Reader
|
var body io.Reader
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
raw, err := json.Marshal(payload)
|
raw, err := json.Marshal(payload)
|
||||||
@@ -100,7 +160,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.SetBasicAuth(c.User, password)
|
if err := c.authorize(req); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
req.Header.Set("Accept", "application/json")
|
req.Header.Set("Accept", "application/json")
|
||||||
if payload != nil {
|
if payload != nil {
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
@@ -118,6 +180,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
|
|||||||
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
|
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
|
case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "":
|
||||||
|
return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w",
|
||||||
|
method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"})
|
||||||
case res.StatusCode == http.StatusNotFound:
|
case res.StatusCode == http.StatusNotFound:
|
||||||
return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound)
|
return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound)
|
||||||
case res.StatusCode >= 300:
|
case res.StatusCode >= 300:
|
||||||
|
|||||||
+32
-7
@@ -11,7 +11,8 @@ import type { CommitStatus, GiteaClient } from "./client.js";
|
|||||||
import { mergeableUnder, newestByContext } from "./checks.js";
|
import { mergeableUnder, newestByContext } from "./checks.js";
|
||||||
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
|
||||||
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
|
||||||
import { announcementOf, askedWords, outOfDateWords, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js";
|
import { groupRefusal, memberKey, namesSiblings, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js";
|
||||||
|
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js";
|
||||||
|
|
||||||
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
|
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
|
||||||
const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/;
|
const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/;
|
||||||
@@ -54,9 +55,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
const method = args.method ? String(args.method) : "merge";
|
const method = args.method ? String(args.method) : "merge";
|
||||||
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
|
||||||
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
|
||||||
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left
|
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the
|
||||||
// to the forge, whose administrator — the account this tool's token is minted for — may merge past a
|
// forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge
|
||||||
// rule that lets it. A rule that cannot be read is no rule passed.
|
// applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that
|
||||||
|
// cannot be read is no rule passed.
|
||||||
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
|
||||||
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
|
||||||
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
// head was and nothing told it which head the review had approved. The caller names it; a head that moved
|
||||||
@@ -80,7 +82,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
|
||||||
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
|
||||||
}
|
}
|
||||||
const rule = await gitea.branchProtection(owner, repo, pull.base);
|
const rule = await gitea.appliedProtection(owner, repo, pull.base);
|
||||||
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a
|
||||||
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
// branch nothing guards would merge past every check (novox/hq ADR 0266).
|
||||||
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
|
||||||
@@ -96,7 +98,11 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
const checks = await gitea.headChecks(owner, repo, pull.head_sha);
|
const checks = await gitea.headChecks(owner, repo, pull.head_sha);
|
||||||
const at = verdictAtTrunk(checks, trunkHead);
|
const at = verdictAtTrunk(checks, trunkHead);
|
||||||
if (!at.current) {
|
if (!at.current) {
|
||||||
const judged = at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
|
// A verdict put out of date by a sibling's move says so in its own words (ADR 0382 §5); it is asked again as a
|
||||||
|
// trunk's move is, and a failed ask puts those words back, not the trunk's.
|
||||||
|
const siblingWords = siblingOutOfDate(checks[CHECK_CONTEXT]?.description) ? checks[CHECK_CONTEXT]?.description : undefined;
|
||||||
|
const judged = siblingWords ? `was put out of date by a sibling's move (${siblingWords})`
|
||||||
|
: at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
|
||||||
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
|
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
|
||||||
const head = twelve(pull.head_sha), now = twelve(trunkHead);
|
const head = twelve(pull.head_sha), now = twelve(trunkHead);
|
||||||
if (at.onItsWay) {
|
if (at.onItsWay) {
|
||||||
@@ -122,7 +128,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
const cloneURL = (await gitea.getRepo(owner, repo)).clone_url;
|
const cloneURL = (await gitea.getRepo(owner, repo)).clone_url;
|
||||||
await deps.emit("pull.updated", await announcementOf(gitea, owner, repo, cloneURL, pull));
|
await deps.emit("pull.updated", await announcementOf(gitea, owner, repo, cloneURL, pull));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
await say(outOfDateWords(at.judged, trunkHead)).catch(() => {});
|
await say(siblingWords ?? outOfDateWords(at.judged, trunkHead)).catch(() => {});
|
||||||
throw new Error(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
throw new Error(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
|
||||||
`${now}, and its check could not be asked: ${err instanceof Error ? err.message : err}. Nothing was merged`);
|
`${now}, and its check could not be asked: ${err instanceof Error ? err.message : err}. Nothing was merged`);
|
||||||
}
|
}
|
||||||
@@ -138,6 +144,25 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
|
|||||||
throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` +
|
throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` +
|
||||||
`its checks succeed; no tool merges past them (novox/hq ADR 0266)`);
|
`its checks succeed; no tool merges past them (novox/hq ADR 0266)`);
|
||||||
}
|
}
|
||||||
|
// **A member judged beside its siblings merges only with its group passed and reviewed** (novox/hq ADR 0382 §6): the
|
||||||
|
// siblings its verdict names are the holder's record (siblings.ts); a status that says it read siblings with no record
|
||||||
|
// of them is no verdict to merge on, never read as none.
|
||||||
|
const heard = readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)];
|
||||||
|
const siblings = heard && sameCommit(heard.head, pull.head_sha) ? heard.siblings : [];
|
||||||
|
if (!siblings.length && namesSiblings(statuses.find((s) => s.context === CHECK_CONTEXT)?.description)) {
|
||||||
|
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict read siblings, and which ones is not held ` +
|
||||||
|
`here, so the group cannot be weighed. Ask its check again at ${twelve(pull.head_sha)}. Nothing was merged`);
|
||||||
|
}
|
||||||
|
if (siblings.length) {
|
||||||
|
const missing = await groupRefusal({ headChecks: (o, r, sha) => gitea.headChecks(o, r, sha),
|
||||||
|
approvedAt: (o, r, n, h) => gitea.approvedAt(o, r, n, h) },
|
||||||
|
{ owner, repo, number, head: String(pull.head_sha) }, siblings);
|
||||||
|
if (missing) {
|
||||||
|
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict was judged beside ${siblings.length} ` +
|
||||||
|
`sibling(s), and ${missing}. A member judged beside its siblings merges once its group passed at the heads that ` +
|
||||||
|
`verdict read and each of them is approved there (novox/hq ADR 0382). Nothing was merged`);
|
||||||
|
}
|
||||||
|
}
|
||||||
await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha);
|
await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha);
|
||||||
// The merge commit only exists now; answered so the caller can follow what is built from it.
|
// The merge commit only exists now; answered so the caller can follow what is built from it.
|
||||||
// pull.merged is the events entrypoint's to announce (index.ts), once, within its poll.
|
// pull.merged is the events entrypoint's to announce (index.ts), once, within its poll.
|
||||||
|
|||||||
@@ -159,8 +159,8 @@
|
|||||||
"settings": {
|
"settings": {
|
||||||
"ready-authors": {
|
"ready-authors": {
|
||||||
"kind": "preference",
|
"kind": "preference",
|
||||||
"default": "mesh-admin jschoubben",
|
"default": "mesh-tools mesh-admin jschoubben",
|
||||||
"why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the mesh's own forge account and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word"
|
"why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"resources": [
|
"resources": [
|
||||||
@@ -184,6 +184,13 @@
|
|||||||
"trusted": true,
|
"trusted": true,
|
||||||
"content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n"
|
"content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"id": "forge-patches",
|
||||||
|
"type": "directory",
|
||||||
|
"path": "/var/lib/mesh/forge-patches",
|
||||||
|
"mode": "0733",
|
||||||
|
"owner": "${machine:account}"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"id": "state",
|
"id": "state",
|
||||||
"type": "directory",
|
"type": "directory",
|
||||||
@@ -308,6 +315,7 @@
|
|||||||
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json",
|
||||||
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
||||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
|
||||||
|
"MESH_GITEA_TOOLS_USER": "mesh-tools",
|
||||||
"MESH_GITEA_STATE_DIR": "${dir:runtime-state}",
|
"MESH_GITEA_STATE_DIR": "${dir:runtime-state}",
|
||||||
"MESH_RECEIVES": "${dir:grants}/npm.json",
|
"MESH_RECEIVES": "${dir:grants}/npm.json",
|
||||||
"MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors",
|
"MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors",
|
||||||
@@ -345,8 +353,9 @@
|
|||||||
],
|
],
|
||||||
"env": {
|
"env": {
|
||||||
"MESH_GITEA_URL": "http://127.0.0.1:${port:3000}",
|
"MESH_GITEA_URL": "http://127.0.0.1:${port:3000}",
|
||||||
"MESH_GITEA_ADMIN_USER": "mesh-admin",
|
"MESH_GITEA_TOOLS_USER": "mesh-tools",
|
||||||
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret"
|
"MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token",
|
||||||
|
"MESH_GITEA_PATCH_DIR": "${dir:forge-patches}"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|||||||
+42
-4
@@ -9,8 +9,9 @@
|
|||||||
//
|
//
|
||||||
// Pure functions here, so they are tested without a forge; index.ts does the asking and the setting.
|
// Pure functions here, so they are tested without a forge; index.ts does the asking and the setting.
|
||||||
|
|
||||||
import type { CommitStatus, GiteaPull, GiteaRepo } from "./client.js";
|
import type { CommitStatus, GiteaClient, GiteaPull, GiteaRepo } from "./client.js";
|
||||||
import { movedSince } from "./client.js";
|
import { movedSince } from "./client.js";
|
||||||
|
import { UNKNOWN_MARK, besideWords, siblingsOf } from "./siblings.js";
|
||||||
|
|
||||||
/** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */
|
/** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */
|
||||||
export const CHECK_CONTEXT = "mesh/merge-gate";
|
export const CHECK_CONTEXT = "mesh/merge-gate";
|
||||||
@@ -65,6 +66,13 @@ export interface Checked {
|
|||||||
trunk?: string;
|
trunk?: string;
|
||||||
beside?: Record<string, string>;
|
beside?: Record<string, string>;
|
||||||
took?: string;
|
took?: string;
|
||||||
|
/** The other members of its delivery group the check read beside the head, each at its head merged onto its base
|
||||||
|
* commit (novox/hq ADR 0382 §4); absent when it read none, and from a controller from before the rule. */
|
||||||
|
siblings?: { repository: string; number: number; head: string; base: string; base_commit: string }[];
|
||||||
|
/** Why the siblings could not be learned (mesh-delivery did not answer), when that is so. */
|
||||||
|
siblings_unknown?: string;
|
||||||
|
/** Where the judge came from, when it came from the group (ADR 0382 §3): said in the account. */
|
||||||
|
judge_from?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A change plan, as the controller says it (mesh-controller internal/link, ChangePlan). */
|
/** A change plan, as the controller says it (mesh-controller internal/link, ChangePlan). */
|
||||||
@@ -163,7 +171,7 @@ function described(verdict: string, summary: string, modules?: string[], depende
|
|||||||
return clipped(d);
|
return clipped(d);
|
||||||
}
|
}
|
||||||
|
|
||||||
function clipped(d: string): string {
|
export function clipped(d: string): string {
|
||||||
d = d.replace(/\s+/g, " ").trim();
|
d = d.replace(/\s+/g, " ").trim();
|
||||||
return d.length > 140 ? d.slice(0, 139) + "…" : d;
|
return d.length > 140 ? d.slice(0, 139) + "…" : d;
|
||||||
}
|
}
|
||||||
@@ -176,6 +184,24 @@ export interface RepoCheckFacts {
|
|||||||
required?: boolean;
|
required?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
|
||||||
|
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
|
||||||
|
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
|
||||||
|
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
|
||||||
|
// required would let an untested repository merge. The base's protection is the one the forge applies, read
|
||||||
|
// from the branch: the rules' own list is a repository administrator's, which the tools' account is not
|
||||||
|
// (novox/hq ADR 0365).
|
||||||
|
export async function repoCheckFacts(
|
||||||
|
forge: Pick<GiteaClient, "holdsFile" | "appliedProtection">, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
|
||||||
|
if (c["repo-check"]?.verdict !== "warning") return {};
|
||||||
|
const defined = await forge.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
|
||||||
|
if (defined !== false) return { defined };
|
||||||
|
const rule = await forge.appliedProtection(c.owner, c.repo, base || "main").catch(() => undefined);
|
||||||
|
if (rule === undefined) return { defined };
|
||||||
|
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
|
||||||
|
return { defined, required };
|
||||||
|
}
|
||||||
|
|
||||||
/** The description of a required repository check that is not defined. */
|
/** The description of a required repository check that is not defined. */
|
||||||
export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " +
|
export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " +
|
||||||
"a person adds one, or lifts the requirement from the branch's protection";
|
"a person adds one, or lifts the requirement from the branch's protection";
|
||||||
@@ -220,8 +246,12 @@ export function statusesFor(c: Checked, facts: RepoCheckFacts = {}): CommitStatu
|
|||||||
* 7a8b9c0d1e2f — pass: …", so the forge's 140 characters never clip it away; the trunk's move is weighed by it. A
|
* 7a8b9c0d1e2f — pass: …", so the forge's 140 characters never clip it away; the trunk's move is weighed by it. A
|
||||||
* verdict that names no trunk commit is said as before. */
|
* verdict that names no trunk commit is said as before. */
|
||||||
function judgedFirst(c: Checked, status: CommitStatus): CommitStatus {
|
function judgedFirst(c: Checked, status: CommitStatus): CommitStatus {
|
||||||
if (!c.trunk) return status;
|
// The siblings it read, named after the commits (ADR 0382 §4), so the clip never takes them either.
|
||||||
return { ...status, description: clipped(`head ${c.commit.slice(0, 12)} merged into trunk ${c.trunk.slice(0, 12)} — ${status.description}`) };
|
const siblings = siblingsOf(c.siblings);
|
||||||
|
// A verdict that could not learn them says so on the status too (clause 1: it is still checked, and merges).
|
||||||
|
const mark = siblings.length ? besideWords(siblings) : c.siblings_unknown ? UNKNOWN_MARK : "";
|
||||||
|
if (!c.trunk) return mark ? { ...status, description: clipped(`${mark} — ${status.description}`) } : status;
|
||||||
|
return { ...status, description: clipped(`head ${c.commit.slice(0, 12)} merged into trunk ${c.trunk.slice(0, 12)}${mark ? " " + mark : ""} — ${status.description}`) };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The comment a verdict leaves on its pull request, with the check's own account: the change plan of a
|
/** The comment a verdict leaves on its pull request, with the check's own account: the change plan of a
|
||||||
@@ -240,6 +270,14 @@ export function commentFor(c: Checked, facts: RepoCheckFacts = {}): string | nul
|
|||||||
lines.splice(1, 0, `Judged: head \`${c.commit.slice(0, 12)}\` merged into trunk \`${c.trunk.slice(0, 12)}\`` +
|
lines.splice(1, 0, `Judged: head \`${c.commit.slice(0, 12)}\` merged into trunk \`${c.trunk.slice(0, 12)}\`` +
|
||||||
(beside ? `; read beside it: ${beside}` : ""));
|
(beside ? `; read beside it: ${beside}` : ""));
|
||||||
}
|
}
|
||||||
|
const siblings = siblingsOf(c.siblings);
|
||||||
|
if (siblings.length) {
|
||||||
|
lines.splice(c.trunk ? 2 : 1, 0, `Siblings read beside it: ` + siblings.map((s) =>
|
||||||
|
`${s.repository}#${s.number} \`${s.head.slice(0, 12)}\` merged onto \`${s.base_commit.slice(0, 12)}\` (${s.base})`).join(", ") +
|
||||||
|
(c.judge_from ? `; judged by ${c.judge_from}` : ""));
|
||||||
|
} else if (c.siblings_unknown) {
|
||||||
|
lines.splice(c.trunk ? 2 : 1, 0, `Siblings could not be learned: ${c.siblings_unknown}`);
|
||||||
|
}
|
||||||
lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` +
|
lines.push(`- \`${CHECK_CONTEXT}\`: **${saidAs(gate.verdict).toUpperCase()}** — ${gate.summary}` +
|
||||||
(gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` +
|
(gate.modules?.length ? ` (modules: ${gate.modules.join(", ")}` +
|
||||||
(gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : ""));
|
(gate.dependents?.length ? `; built after them: ${gate.dependents.join(", ")}` : "") + ")" : ""));
|
||||||
|
|||||||
@@ -0,0 +1,286 @@
|
|||||||
|
// **A delivery group's member is checked beside the other members' heads, and a sibling's move puts that verdict out of
|
||||||
|
// date** (novox/hq ADR 0382 §5, §6; issue 867).
|
||||||
|
//
|
||||||
|
// The controller's verdict (`checked`) names the siblings its check read: each as the sibling's repository, pull
|
||||||
|
// request, head, base branch and the base commit the sibling's head was merged onto. This module, the forge's holder,
|
||||||
|
// keeps them per open pull request, beside its other records, and:
|
||||||
|
//
|
||||||
|
// - on its own hearing of a sibling's new head, of its close unmerged, or of a move of its base, sets
|
||||||
|
// mesh/merge-gate and mesh/repo-check of every member whose verdict names that sibling back to pending, with the
|
||||||
|
// reason. Nothing is run for it. The one move that does not count is the sibling's own merge of the head that was
|
||||||
|
// read onto the base commit that was read (in either order of the two merges);
|
||||||
|
// - lets the merge tool refuse a verdict that read siblings unless mesh/delivery-group passed at the very heads it
|
||||||
|
// read, and every sibling head it read carries an approving review at that head (merge.ts).
|
||||||
|
//
|
||||||
|
// A verdict without siblings (a controller from before ADR 0382) is none of this: nothing is kept and nothing changes.
|
||||||
|
// Pure functions and one small file of the module's own state, so this is tested without a forge.
|
||||||
|
|
||||||
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
||||||
|
import { dirname } from "node:path";
|
||||||
|
import type { CommitStatus, GiteaPull, HeadCheck } from "./client.js";
|
||||||
|
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, clipped } from "./pulls.js";
|
||||||
|
import { sameCommit, twelve, type OutOfDateWriter } from "./trunk.js";
|
||||||
|
|
||||||
|
/** The status the delivery group's owner sets on each member's head (novox/hq ADR 0239). */
|
||||||
|
export const GROUP_CONTEXT = "mesh/delivery-group";
|
||||||
|
|
||||||
|
/** One sibling a verdict read, as the controller names it (contract of ADR 0382 §4). */
|
||||||
|
export interface SiblingNamed {
|
||||||
|
repository: string;
|
||||||
|
number: number;
|
||||||
|
head: string;
|
||||||
|
base: string;
|
||||||
|
base_commit: string;
|
||||||
|
/** Set here, never by the controller: the sibling merged its head onto its base commit read, so it is no sibling's
|
||||||
|
* move any more. */
|
||||||
|
merged?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What is kept of one member's latest verdict that read siblings: the head it was of and the siblings it named. */
|
||||||
|
export interface SiblingEntry {
|
||||||
|
head: string;
|
||||||
|
siblings: SiblingNamed[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The records, by `owner/repo#number` of the member. */
|
||||||
|
export type SiblingRecord = Record<string, SiblingEntry>;
|
||||||
|
|
||||||
|
/** The member's key. */
|
||||||
|
export function memberKey(owner: string, repo: string, number: number): string {
|
||||||
|
return `${owner}/${repo}#${number}`.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The siblings a verdict names, trimmed to what is kept; none when it names none or names one badly. */
|
||||||
|
export function siblingsOf(named: unknown): SiblingNamed[] {
|
||||||
|
if (!Array.isArray(named)) return [];
|
||||||
|
const out: SiblingNamed[] = [];
|
||||||
|
for (const s of named) {
|
||||||
|
const r = s as Record<string, unknown>;
|
||||||
|
const repository = String(r?.repository ?? ""), head = String(r?.head ?? ""), base = String(r?.base ?? "");
|
||||||
|
const number = Number(r?.number);
|
||||||
|
if (!repository || !head || !base || !Number.isInteger(number) || number <= 0) continue;
|
||||||
|
out.push({ repository, number, head, base, base_commit: String(r?.base_commit ?? "") });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The siblings as a status names them: "mesh-host#114@abcdef123456, …". */
|
||||||
|
export function siblingWords(siblings: SiblingNamed[]): string {
|
||||||
|
return siblings.map((s) => `${s.repository.split("/").pop()}#${s.number}@${twelve(s.head)}`).join(", ");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What a status of a verdict that read siblings carries, after the commits it judged. */
|
||||||
|
export function besideWords(siblings: SiblingNamed[]): string {
|
||||||
|
// Short, so that the verdict word and the start of its reason survive the forge's 140 characters: the first sibling
|
||||||
|
// by its pull request and 8 characters of its head, and how many more. The whole list is the holder's record and the comment's.
|
||||||
|
const first = siblings[0];
|
||||||
|
const more = siblings.length > 1 ? ` +${siblings.length - 1} more` : "";
|
||||||
|
return `beside ${first.repository.split("/").pop()}#${first.number}@${twelve(first.head).slice(0, 8)}${more}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The mark a status carries when the siblings could not be learned (ADR 0382 §1): said, never silent. */
|
||||||
|
export const UNKNOWN_MARK = "siblings unknown";
|
||||||
|
|
||||||
|
/** Whether a status names that its verdict read siblings (so that a record lost does not read as none). */
|
||||||
|
export function namesSiblings(description: string | undefined): boolean {
|
||||||
|
return / beside [\w.-]+#\d+@[0-9a-f]{8}\b/.test(description ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a status is a verdict put out of date by a sibling's move. */
|
||||||
|
export function siblingOutOfDate(description: string | undefined): boolean {
|
||||||
|
return /^out of date: judged beside /.test(description ?? "");
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the record on disk ----------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** The file the holder keeps the siblings its verdicts name in, beside its other records. */
|
||||||
|
export function siblingsFile(stateDir: string | undefined): string | null {
|
||||||
|
return stateDir ? `${stateDir.replace(/\/+$/, "")}/siblings-named.json` : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readSiblings(file: string | null): SiblingRecord {
|
||||||
|
if (!file || !existsSync(file)) return {};
|
||||||
|
try {
|
||||||
|
return (JSON.parse(readFileSync(file, "utf8")) as { members?: SiblingRecord }).members ?? {};
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function keepSiblings(file: string | null, record: SiblingRecord): void {
|
||||||
|
if (!file) return;
|
||||||
|
try {
|
||||||
|
mkdirSync(dirname(file), { recursive: true });
|
||||||
|
writeFileSync(file + ".tmp", JSON.stringify({ members: record }));
|
||||||
|
renameSync(file + ".tmp", file);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] the siblings a verdict named could not be kept — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A member's latest verdict, kept: its siblings when it read some, else the entry removed. Answers whether it changed. */
|
||||||
|
export function noteVerdict(record: SiblingRecord, key: string, head: string, siblings: SiblingNamed[]): boolean {
|
||||||
|
if (siblings.length === 0) {
|
||||||
|
if (!(key in record)) return false;
|
||||||
|
delete record[key];
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
record[key] = { head, siblings };
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- what a sibling's move is ---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** What weighing a sibling reads of the forge. */
|
||||||
|
export interface SiblingForge {
|
||||||
|
getPullRequest(owner: string, repo: string, index: number): Promise<GiteaPull>;
|
||||||
|
branchHead(owner: string, repo: string, branch: string): Promise<string>;
|
||||||
|
commitParents(owner: string, repo: string, sha: string): Promise<string[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The reason a verdict that named this sibling is out of date, from what the forge holds now, or nothing; and whether
|
||||||
|
* the sibling has merged its head read onto the base commit read, which ends it as a sibling. */
|
||||||
|
export async function siblingMoved(forge: SiblingForge, s: SiblingNamed): Promise<{ reason?: string; merged?: boolean }> {
|
||||||
|
const [owner, repo] = s.repository.split("/");
|
||||||
|
const who = `${repo}#${s.number}`;
|
||||||
|
const pull = await forge.getPullRequest(owner, repo, s.number);
|
||||||
|
if (pull.merged) {
|
||||||
|
// Its own merge of the head that was read onto the base commit that was read: the head the merge joined is the head
|
||||||
|
// read, and the commit it was made onto is the base commit read. Any other merge is a move of the base.
|
||||||
|
const parents = pull.merge_commit_sha ? await forge.commitParents(owner, repo, pull.merge_commit_sha).catch(() => []) : [];
|
||||||
|
const own = sameCommit(pull.head_sha, s.head) && parents.length >= 1 && sameCommit(parents[0], s.base_commit) &&
|
||||||
|
(parents.length === 1 || sameCommit(parents[1], s.head));
|
||||||
|
if (own) return { merged: true };
|
||||||
|
return { reason: `${who} was merged otherwise than its head ${twelve(s.head)} onto ${twelve(s.base_commit)}` };
|
||||||
|
}
|
||||||
|
if (pull.state !== "open") return { reason: `${who} was closed unmerged` };
|
||||||
|
if (!sameCommit(pull.head_sha, s.head)) {
|
||||||
|
return { reason: `${who} at ${twelve(s.head)}, it is now at ${twelve(pull.head_sha)}` };
|
||||||
|
}
|
||||||
|
const base = await forge.branchHead(owner, repo, s.base);
|
||||||
|
if (!sameCommit(base, s.base_commit)) {
|
||||||
|
return { reason: `${who} merged onto ${twelve(s.base_commit)}, its ${s.base} is now ${twelve(base)}` };
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A verdict out of date because a sibling moved, as its statuses say it (ADR 0382 §5; the wording of ADR 0323 §4). */
|
||||||
|
export function siblingOutOfDateWords(reason: string): string {
|
||||||
|
return clipped(`out of date: judged beside ${reason}; judged again on a recheck or when it is merged`);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The statuses a sibling's move puts back to pending on one member's head, from what the forge holds on it now: none
|
||||||
|
* when it has no merge check, or its gate is pending (a check on its way, or already out of date). */
|
||||||
|
export function siblingStatuses(checks: Record<string, HeadCheck>, reason: string): CommitStatus[] {
|
||||||
|
const gate = checks[CHECK_CONTEXT];
|
||||||
|
if (!gate || gate.state === "pending") return [];
|
||||||
|
const description = siblingOutOfDateWords(reason);
|
||||||
|
const out: CommitStatus[] = [{ state: "pending", context: CHECK_CONTEXT, description }];
|
||||||
|
if (checks[REPO_CHECK_CONTEXT]) out.push({ state: "pending", context: REPO_CHECK_CONTEXT, description });
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What weighing a sibling's move writes with, beside the forge's reads. */
|
||||||
|
export interface SiblingWeighing extends SiblingForge {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A sibling moved (a new head, a close, a base move, a merge): every member whose latest verdict names it has its
|
||||||
|
* merge check put back to pending with the reason. `repository` and, when the event names one, `number` say which
|
||||||
|
* sibling. A member whose head is no longer the one its verdict was of is dropped from the record: its new head is
|
||||||
|
* checked afresh. Nothing is run. Answers whether the record changed. A read that fails leaves that member for the next
|
||||||
|
* move to weigh, and is said.
|
||||||
|
*/
|
||||||
|
export async function weighSiblings(forge: SiblingWeighing, gates: OutOfDateWriter, record: SiblingRecord,
|
||||||
|
repository: string, number?: number): Promise<boolean> {
|
||||||
|
let changed = false;
|
||||||
|
const repo = repository.toLowerCase();
|
||||||
|
for (const [key, entry] of Object.entries(record)) {
|
||||||
|
const [path] = key.split("#");
|
||||||
|
const [owner, name] = path.split("/");
|
||||||
|
const memberNumber = Number(key.split("#")[1]);
|
||||||
|
// A member that is merged or no longer open has no verdict to put out of date: its entry goes (the record must not
|
||||||
|
// grow without bound, and a merged head is never re-pended by a sibling's later push). Weighed when the event is the
|
||||||
|
// member's own, or names one of its siblings.
|
||||||
|
const own = path === repo && (number === undefined || number === memberNumber);
|
||||||
|
if (own || entry.siblings.some((s) => !s.merged && s.repository.toLowerCase() === repo && (number === undefined || s.number === number))) {
|
||||||
|
try {
|
||||||
|
const member = await forge.getPullRequest(owner, name, memberNumber);
|
||||||
|
if (member.merged || member.state !== "open") {
|
||||||
|
delete record[key];
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] ${key}: whether it is still open could not be read — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const s of entry.siblings) {
|
||||||
|
if (s.merged || s.repository.toLowerCase() !== repo || (number !== undefined && s.number !== number)) continue;
|
||||||
|
try {
|
||||||
|
const moved = await siblingMoved(forge, s);
|
||||||
|
if (moved.merged) {
|
||||||
|
s.merged = true;
|
||||||
|
changed = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!moved.reason) continue;
|
||||||
|
const member = await forge.getPullRequest(owner, name, memberNumber);
|
||||||
|
if (!sameCommit(member.head_sha, entry.head)) {
|
||||||
|
delete record[key];
|
||||||
|
changed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
const written = await gates.outOfDate(owner, name, entry.head, (checks) => siblingStatuses(checks, moved.reason!),
|
||||||
|
member.html_url || undefined);
|
||||||
|
if (written.length) {
|
||||||
|
console.log(`[gitea] ${key}: its verdict was judged beside ${moved.reason}: out of date, nothing run (novox/hq ADR 0382)`);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[gitea] ${key}: whether ${s.repository}#${s.number} moved could not be weighed, so it is weighed again ` +
|
||||||
|
`on its next move — ${err instanceof Error ? err.message : err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- the merge tool's rule (ADR 0382 §6) ----------------------------------------------------------------------
|
||||||
|
|
||||||
|
/** What the rule reads beside the pull request being merged. */
|
||||||
|
export interface GroupForge {
|
||||||
|
headChecks(owner: string, repo: string, sha: string): Promise<Record<string, HeadCheck>>;
|
||||||
|
/** Whether the sibling's head carries an approving review at that head. */
|
||||||
|
approvedAt(owner: string, repo: string, number: number, head: string): Promise<boolean>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What stops a pull request whose verdict read siblings from merging, in words, or nothing: mesh/delivery-group must be
|
||||||
|
* a success at its own head and at every sibling head the verdict read, and every sibling head read must carry an
|
||||||
|
* approving review at that head. Every missing thing is named.
|
||||||
|
*/
|
||||||
|
export async function groupRefusal(forge: GroupForge, self: { owner: string; repo: string; number: number; head: string },
|
||||||
|
siblings: SiblingNamed[]): Promise<string | undefined> {
|
||||||
|
const missing: string[] = [];
|
||||||
|
const members = [{ repository: `${self.owner}/${self.repo}`, number: self.number, head: self.head, sibling: false },
|
||||||
|
...siblings.map((s) => ({ repository: s.repository, number: s.number, head: s.head, sibling: true }))];
|
||||||
|
for (const m of members) {
|
||||||
|
const [owner, repo] = m.repository.split("/");
|
||||||
|
const at = `${repo}#${m.number}@${twelve(m.head)}`;
|
||||||
|
let group: HeadCheck | undefined;
|
||||||
|
try {
|
||||||
|
group = (await forge.headChecks(owner, repo, m.head))[GROUP_CONTEXT];
|
||||||
|
} catch (err) {
|
||||||
|
missing.push(`${GROUP_CONTEXT} at ${at} could not be read (${err instanceof Error ? err.message : err})`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (group?.state !== "success") missing.push(`${GROUP_CONTEXT} is ${group?.state ?? "missing"} at ${at}`);
|
||||||
|
if (!m.sibling) continue;
|
||||||
|
try {
|
||||||
|
if (!(await forge.approvedAt(owner, repo, m.number, m.head))) missing.push(`${at} carries no approving review at that head`);
|
||||||
|
} catch (err) {
|
||||||
|
missing.push(`whether ${at} carries an approving review could not be read (${err instanceof Error ? err.message : err})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return missing.length ? missing.join("; ") : undefined;
|
||||||
|
}
|
||||||
@@ -47,11 +47,18 @@ test("every open pull request is read with what its base requires and what its h
|
|||||||
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) {
|
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) {
|
||||||
return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" });
|
return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" });
|
||||||
}
|
}
|
||||||
if (url.startsWith("/api/v1/repos/novox/hq/branch_protections")) {
|
// The base's protection, read from the branch as the tools' account may (novox/hq ADR 0365); the rules' list is refused it.
|
||||||
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/repo-check"] }]);
|
if (url.includes("/branch_protections")) {
|
||||||
|
res.statusCode = 403;
|
||||||
|
return res.end(JSON.stringify({ message: "the rules are a repository administrator's" }));
|
||||||
}
|
}
|
||||||
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branch_protections")) {
|
if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) {
|
||||||
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }]);
|
return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
|
||||||
|
status_check_contexts: ["mesh/repo-check"] });
|
||||||
|
}
|
||||||
|
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branches/main")) {
|
||||||
|
return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
|
||||||
|
status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] });
|
||||||
}
|
}
|
||||||
if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
|
if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
|
||||||
if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending",
|
if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending",
|
||||||
@@ -251,6 +258,8 @@ test("a long thread hides nothing: every page of the timeline is read", async ()
|
|||||||
test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => {
|
test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => {
|
||||||
const { readyAuthors } = await import("../client.ts");
|
const { readyAuthors } = await import("../client.ts");
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]);
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]);
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }), ["mesh-admin"]);
|
// The tools' account and the admin's, whose marks were made before ADR 0365 (issue 469).
|
||||||
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n"), ["mesh-admin"]);
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
|
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n", "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
|
assert.deepEqual(readyAuthors(undefined, "mesh-admin", undefined, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -30,6 +30,9 @@ const state = {
|
|||||||
// What the tools said on the bus.
|
// What the tools said on the bus.
|
||||||
emitted: [] as { type: string; body: any }[],
|
emitted: [] as { type: string; body: any }[],
|
||||||
rules: [] as any[],
|
rules: [] as any[],
|
||||||
|
// Whether the token is a repository administrator's. The tools' account is not (novox/hq ADR 0365): the forge
|
||||||
|
// refuses it the rules' list, as gitea's reqAdmin does, and it reads a branch's protection from the branch.
|
||||||
|
admin: false,
|
||||||
lastCommit: "2026-01-01T00:00:00Z",
|
lastCommit: "2026-01-01T00:00:00Z",
|
||||||
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
|
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
|
||||||
// record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the
|
// record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the
|
||||||
@@ -41,8 +44,13 @@ const state = {
|
|||||||
// When set, the merge commit's parents as the forge reports them, or the status it answers them with.
|
// When set, the merge commit's parents as the forge reports them, or the status it answers them with.
|
||||||
parents: null as string[] | null,
|
parents: null as string[] | null,
|
||||||
commitsFail: 0,
|
commitsFail: 0,
|
||||||
|
// The sibling beside which a verdict was judged (novox/hq ADR 0382 §6): its group status at its head, and its reviews.
|
||||||
|
groupStatus: "",
|
||||||
|
siblingGroup: "",
|
||||||
|
reviews: [] as { state: string; commit_id: string; dismissed?: boolean }[],
|
||||||
};
|
};
|
||||||
const REVIEWED = "abc1234def5678901234567890abcdef12345678";
|
const REVIEWED = "abc1234def5678901234567890abcdef12345678";
|
||||||
|
const SIBLING_HEAD = "bbbbbbbbbbbb5678901234567890abcdef12345678".slice(0, 40);
|
||||||
const TRUNK = "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b";
|
const TRUNK = "7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b";
|
||||||
const MOVED = "3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d";
|
const MOVED = "3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d";
|
||||||
// A verdict judged against a trunk commit, as both statuses say it (novox/hq ADR 0323 §3).
|
// A verdict judged against a trunk commit, as both statuses say it (novox/hq ADR 0323 §3).
|
||||||
@@ -59,12 +67,16 @@ beforeEach(() => {
|
|||||||
state.trunk = TRUNK;
|
state.trunk = TRUNK;
|
||||||
state.emitted = [];
|
state.emitted = [];
|
||||||
state.rules = [protectedMain()];
|
state.rules = [protectedMain()];
|
||||||
|
state.admin = false;
|
||||||
state.head = REVIEWED;
|
state.head = REVIEWED;
|
||||||
state.landed = "";
|
state.landed = "";
|
||||||
state.merged = false;
|
state.merged = false;
|
||||||
state.mergeCommit = "feedface00000000000000000000000000000000";
|
state.mergeCommit = "feedface00000000000000000000000000000000";
|
||||||
state.parents = null;
|
state.parents = null;
|
||||||
state.commitsFail = 0;
|
state.commitsFail = 0;
|
||||||
|
state.groupStatus = "";
|
||||||
|
state.siblingGroup = "";
|
||||||
|
state.reviews = [];
|
||||||
});
|
});
|
||||||
|
|
||||||
const json = (res: ServerResponse, status: number, body: unknown): void => {
|
const json = (res: ServerResponse, status: number, body: unknown): void => {
|
||||||
@@ -87,8 +99,18 @@ const server = createServer(async (req, res) => {
|
|||||||
if (method !== "GET") state.sent.push({ method, path, body });
|
if (method !== "GET") state.sent.push({ method, path, body });
|
||||||
const repo = "/repos/novox/mesh-catalog";
|
const repo = "/repos/novox/mesh-catalog";
|
||||||
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
|
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
|
||||||
|
if (path.startsWith(`${repo}/branch_protections`) && !state.admin) {
|
||||||
|
return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
}
|
||||||
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
|
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
|
||||||
if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } });
|
const branch = method === "GET" && path.startsWith(`${repo}/branches/`) ? decodeURIComponent(path.slice(`${repo}/branches/`.length)) : null;
|
||||||
|
if (branch !== null) {
|
||||||
|
// As the forge says it to any reader of the code: the rule that covers the branch, as the forge matches it.
|
||||||
|
const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch));
|
||||||
|
return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule,
|
||||||
|
...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check,
|
||||||
|
status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) });
|
||||||
|
}
|
||||||
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
if (method === "GET" && path === `${repo}/pulls/7/files`) {
|
||||||
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
|
||||||
}
|
}
|
||||||
@@ -113,8 +135,17 @@ const server = createServer(async (req, res) => {
|
|||||||
return json(res, 200, { sha: state.mergeCommit, parents: parents.map((sha) => ({ sha })) });
|
return json(res, 200, { sha: state.mergeCommit, parents: parents.map((sha) => ({ sha })) });
|
||||||
}
|
}
|
||||||
if (method === "GET" && path === `${repo}/commits/${state.head}/status`) {
|
if (method === "GET" && path === `${repo}/commits/${state.head}/status`) {
|
||||||
return json(res, 200, { sha: state.head, state: "success", statuses: state.statuses });
|
const group = state.groupStatus ? [{ context: "mesh/delivery-group", status: state.groupStatus }] : [];
|
||||||
|
return json(res, 200, { sha: state.head, state: "success", statuses: [...state.statuses, ...group] });
|
||||||
}
|
}
|
||||||
|
if (method === "GET" && path === `/repos/novox/mesh-host/commits/${SIBLING_HEAD}/status`) {
|
||||||
|
return json(res, 200, { sha: SIBLING_HEAD, state: "success",
|
||||||
|
statuses: state.siblingGroup ? [{ context: "mesh/delivery-group", status: state.siblingGroup }] : [] });
|
||||||
|
}
|
||||||
|
if (method === "GET" && path === "/repos/novox/mesh-host/pulls/114/reviews") {
|
||||||
|
return json(res, 200, url.searchParams.get("page") === "1" ? state.reviews : []);
|
||||||
|
}
|
||||||
|
if (method === "GET" && path === "/repos/novox/mesh-host/issues/114/labels") return json(res, 200, []);
|
||||||
if (method === "POST" && path === `${repo}/pulls/7/merge`) {
|
if (method === "POST" && path === `${repo}/pulls/7/merge`) {
|
||||||
state.merged = true;
|
state.merged = true;
|
||||||
return json(res, 200, null);
|
return json(res, 200, null);
|
||||||
@@ -199,6 +230,7 @@ test("a protection rule covers a branch as the forge decides it: classes, altern
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_api still writes what an author writes", async () => {
|
test("gitea_api still writes what an author writes", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } });
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });
|
||||||
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" });
|
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" });
|
||||||
@@ -206,6 +238,7 @@ test("gitea_api still writes what an author writes", async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_branch_protection_set only strengthens", async () => {
|
test("gitea_branch_protection_set only strengthens", async () => {
|
||||||
|
state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge
|
||||||
const set = (args: Record<string, unknown>) =>
|
const set = (args: Record<string, unknown>) =>
|
||||||
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
|
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
|
||||||
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
|
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
|
||||||
@@ -396,6 +429,7 @@ test("gitea_merge_pull_request reads back a fast-forward, and says it cannot rea
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => {
|
test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm });
|
const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm });
|
||||||
await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/);
|
await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/);
|
||||||
await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/);
|
await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/);
|
||||||
@@ -427,6 +461,7 @@ test("gitea_merge_pull_request merges only into a branch whose protection requir
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
|
||||||
|
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
|
||||||
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
|
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
|
||||||
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
|
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
|
||||||
const set = (args: Record<string, unknown>) =>
|
const set = (args: Record<string, unknown>) =>
|
||||||
@@ -511,3 +546,64 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud
|
|||||||
assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x",
|
assert.deepEqual(state.asked, ["DELETE /repos/novox/mesh-catalog/branches/feat/x",
|
||||||
"GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]);
|
"GET /repos/novox/mesh-catalog/contents/modules/gitea/module.json"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
// **A member judged beside its siblings merges only with its group passed and reviewed** (novox/hq ADR 0382 §6, issue 867):
|
||||||
|
// the verdict names the sibling; mesh/delivery-group must be a success at the member's head and the sibling's head that
|
||||||
|
// verdict read, and the sibling's head must carry an approving review. A verdict that read none merges as before.
|
||||||
|
test("gitea_merge_pull_request merges a verdict that read siblings only with its group passed and each sibling head reviewed", async () => {
|
||||||
|
const { keepSiblings, siblingsFile } = await import("../dist/siblings.js");
|
||||||
|
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||||
|
const beside = `head ${REVIEWED.slice(0, 12)} merged into trunk ${TRUNK.slice(0, 12)} beside mesh-host#114@${SIBLING_HEAD.slice(0, 8)}`;
|
||||||
|
const named = (siblings: unknown[]) => keepSiblings(siblingsFile(stateDir),
|
||||||
|
{ "novox/mesh-catalog#7": { head: REVIEWED, siblings: siblings as any } });
|
||||||
|
state.statuses = [{ context: "mesh/merge-gate", state: "success", description: `${beside} — pass: x` },
|
||||||
|
{ context: "mesh/repo-check", state: "success", description: `${beside} — pass: y` }];
|
||||||
|
named([{ repository: "novox/mesh-host", number: 114, head: SIBLING_HEAD, base: "main", base_commit: "c".repeat(40) }]);
|
||||||
|
|
||||||
|
// Nothing passed: every missing thing is named, nothing merged.
|
||||||
|
await assert.rejects(merge(), (err: Error) => {
|
||||||
|
assert.match(err.message, /is not merged: its verdict was judged beside 1 sibling\(s\), and mesh\/delivery-group is missing at mesh-catalog#7@abc1234def56; mesh\/delivery-group is missing at mesh-host#114@bbbbbbbbbbbb; mesh-host#114@bbbbbbbbbbbb carries no approving review at that head/);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
assert.equal(state.merged, false);
|
||||||
|
|
||||||
|
// The group passed at the member's head only: the sibling's head still lacks it.
|
||||||
|
state.groupStatus = "success";
|
||||||
|
await assert.rejects(merge(), /mesh\/delivery-group is missing at mesh-host#114@bbbbbbbbbbbb; mesh-host#114@bbbbbbbbbbbb carries no approving review/);
|
||||||
|
// Passed at both heads, but the only approval is at another head, or dismissed: not an approval at the head read.
|
||||||
|
state.siblingGroup = "success";
|
||||||
|
state.reviews = [{ state: "APPROVED", commit_id: "d".repeat(40) }, { state: "APPROVED", commit_id: SIBLING_HEAD, dismissed: true },
|
||||||
|
{ state: "COMMENT", commit_id: SIBLING_HEAD }];
|
||||||
|
await assert.rejects(merge(), /judged beside 1 sibling\(s\), and mesh-host#114@bbbbbbbbbbbb carries no approving review at that head\. A member judged beside/);
|
||||||
|
assert.equal(state.merged, false);
|
||||||
|
|
||||||
|
// Passed and approved at the very heads read: it merges.
|
||||||
|
state.reviews.push({ state: "APPROVED", commit_id: SIBLING_HEAD });
|
||||||
|
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||||
|
|
||||||
|
// A status that says it read siblings, with no record of them, is no verdict to merge on.
|
||||||
|
state.merged = false;
|
||||||
|
keepSiblings(siblingsFile(stateDir), {});
|
||||||
|
await assert.rejects(merge(), /its verdict read siblings, and which ones is not held here/);
|
||||||
|
assert.equal(state.merged, false);
|
||||||
|
|
||||||
|
// A verdict that read none (an older controller, or a group of one) merges exactly as before.
|
||||||
|
state.statuses = judgedAt(TRUNK);
|
||||||
|
state.groupStatus = "";
|
||||||
|
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a verdict put out of date by a sibling's move is refused in its own words, and asked again once", async () => {
|
||||||
|
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
|
||||||
|
const words = "out of date: judged beside mesh-host#114 at bbbbbbbbbbbb, it is now at dddddddddddd; judged again on a recheck or when it is merged";
|
||||||
|
state.statuses = judgedAt(TRUNK).map((s) => ({ ...s, state: "pending", description: words }));
|
||||||
|
await assert.rejects(merge(), (err: Error) => {
|
||||||
|
assert.match(err.message, /its verdict was put out of date by a sibling's move \(out of date: judged beside mesh-host#114 at bbbbbbbbbbbb, it is now at dddddddddddd/);
|
||||||
|
assert.match(err.message, /was asked/);
|
||||||
|
assert.match(err.message, /never merges by itself/);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
assert.equal(state.merged, false);
|
||||||
|
assert.equal(state.emitted.length, 1, "the check is asked, once");
|
||||||
|
});
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
// The controller's build-source check asks gitea_branch_protection_get for a repository's rules, which the forge
|
||||||
|
// answers to a repository's administrator alone. The tools' account is not one (novox/hq ADR 0365 §2), so the read
|
||||||
|
// goes with the forge's administrator account, and only the read (ADR 0375, issue 772): a protection edit through the
|
||||||
|
// tools still gets the forge's refusal.
|
||||||
|
|
||||||
|
import { test, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createServer } from "node:http";
|
||||||
|
|
||||||
|
import { GiteaAdmin, GiteaClient } from "../dist/client.js";
|
||||||
|
import { ConfiguredToken } from "../dist/token.js";
|
||||||
|
import { getGiteaTools } from "../dist/tools/index.js";
|
||||||
|
|
||||||
|
const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64");
|
||||||
|
const rules = [
|
||||||
|
{ rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true },
|
||||||
|
{ branch_name: "release", enable_push: false },
|
||||||
|
];
|
||||||
|
const seen: string[] = [];
|
||||||
|
|
||||||
|
const server = createServer((req, res) => {
|
||||||
|
const path = (req.url ?? "").replace(/^\/api\/v1/, "");
|
||||||
|
const as = req.headers.authorization === adminBasic ? "admin" : "tools";
|
||||||
|
seen.push(`${req.method} ${path} as ${as}`);
|
||||||
|
const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); };
|
||||||
|
if (path.includes("/branch_protections")) {
|
||||||
|
if (as !== "admin") return send(403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
if (req.method === "GET") return send(200, rules);
|
||||||
|
}
|
||||||
|
send(404, { message: "not found" });
|
||||||
|
});
|
||||||
|
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||||
|
after(() => new Promise<void>((r) => server.close(() => r())));
|
||||||
|
const { port } = server.address() as { port: number };
|
||||||
|
const url = `http://127.0.0.1:${port}`;
|
||||||
|
const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret"));
|
||||||
|
const tool = (name: string) => tools.find((t) => t.name === name)!;
|
||||||
|
|
||||||
|
test("gitea_branch_protection_get answers every rule, read with the administrator account the tools' token cannot read with", async () => {
|
||||||
|
const out: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller" });
|
||||||
|
assert.deepEqual(out.rules.map((r: any) => r.rule), ["main", "release"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("gitea_branch_protection_get answers the rule for a branch, null when it has none", async () => {
|
||||||
|
const main: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "main" });
|
||||||
|
assert.equal(main.branch, "main");
|
||||||
|
assert.equal(main.rule.rule, "main");
|
||||||
|
assert.equal(main.rule.admin_may_override, false);
|
||||||
|
const none: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "dev" });
|
||||||
|
assert.equal(none.rule, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => {
|
||||||
|
await assert.rejects(
|
||||||
|
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }),
|
||||||
|
/403/,
|
||||||
|
);
|
||||||
|
assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n"));
|
||||||
|
});
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
// **A delivery group's member is judged beside its siblings, and a sibling's move puts that verdict out of date**
|
||||||
|
// (novox/hq ADR 0382 §4-§6, issue 867): the holder keeps the siblings each verdict names; a sibling's new head, close
|
||||||
|
// unmerged or base move (other than its own merge of the head that was read) sets both statuses of every member whose
|
||||||
|
// verdict names it back to pending, and runs nothing; a member merges only with its group passed and every sibling head
|
||||||
|
// read approved.
|
||||||
|
|
||||||
|
import { test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { mkdtempSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { besideWords, groupRefusal, keepSiblings, namesSiblings, noteVerdict, readSiblings, siblingOutOfDate, siblingsFile,
|
||||||
|
siblingsOf, siblingStatuses, weighSiblings, type SiblingRecord } from "../dist/siblings.js";
|
||||||
|
import { weighTrunks } from "../dist/trunk.js";
|
||||||
|
import { commentFor, statusesFor } from "../dist/pulls.js";
|
||||||
|
|
||||||
|
const h = (c: string) => c.repeat(40);
|
||||||
|
const MEMBER_HEAD = h("a"), HOST_HEAD = h("b"), HOST_BASE = h("c"), HOST_NEW = h("d"), HOST_MOVED = h("e"), MERGE = h("f");
|
||||||
|
const sibling = { repository: "novox/mesh-host", number: 114, head: HOST_HEAD, base: "main", base_commit: HOST_BASE };
|
||||||
|
const verdict = { owner: "novox", repo: "mesh-controller", number: 346, commit: MEMBER_HEAD, verdict: "pass", summary: "x", id: "build-1",
|
||||||
|
trunk: h("1").slice(0, 12), siblings: [sibling], judge_from: "sibling novox/mesh-controller#346 merge 123456789abc",
|
||||||
|
gate: { verdict: "pass", summary: "every machine composes ".repeat(10) }, "repo-check": { verdict: "pass", summary: "ok" } };
|
||||||
|
|
||||||
|
test("both statuses name the siblings the verdict read, keeping the verdict word and the start of its reason within 140 characters", () => {
|
||||||
|
// A realistic controller summary for a failure, long enough that the forge clips it.
|
||||||
|
const reason = "2 of 4 machines do not compose: ace: mesh-host.service needs seat node-reads, which no module on ace claims after this change";
|
||||||
|
const c = { ...verdict, gate: { verdict: "fail", summary: reason }, "repo-check": { verdict: "fail", summary: "its merge-check.sh failed: internal/link/check.go:145:2: undefined" } };
|
||||||
|
const [gate, repo] = statusesFor(c);
|
||||||
|
for (const s of [gate, repo]) {
|
||||||
|
assert.ok(s.description.startsWith(`head ${MEMBER_HEAD.slice(0, 12)} merged into trunk 111111111111 beside mesh-host#114@bbbbbbbb — fail: `), s.description);
|
||||||
|
assert.ok(s.description.length <= 140, `${s.description.length} characters`);
|
||||||
|
assert.ok(namesSiblings(s.description));
|
||||||
|
}
|
||||||
|
assert.match(gate.description, /fail: 2 of 4 machines do not compose: ace: mesh-host/, gate.description);
|
||||||
|
assert.match(repo.description, /fail: its merge-check\.sh failed: internal/, repo.description);
|
||||||
|
assert.equal(besideWords([...siblingsOf([sibling]), ...siblingsOf([{ ...sibling, number: 115 }])]), "beside mesh-host#114@bbbbbbbb +1 more");
|
||||||
|
const comment = commentFor({ ...verdict, gate: { verdict: "fail", summary: "no" } })!;
|
||||||
|
assert.match(comment, /Siblings read beside it: novox\/mesh-host#114 `bbbbbbbbbbbb` merged onto `cccccccccccc` \(main\); judged by sibling novox\/mesh-controller#346 merge/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a verdict that could not learn its siblings says so on the status and in the comment, and names none", () => {
|
||||||
|
const { siblings, ...unknown } = { ...verdict, siblings_unknown: "mesh-delivery did not answer" };
|
||||||
|
const reason = "2 of 4 machines do not compose: ace: mesh-host.service needs seat node-reads, which no module claims";
|
||||||
|
for (const s of statusesFor({ ...unknown, gate: { verdict: "fail", summary: reason } })) {
|
||||||
|
assert.match(s.description, /^head aaaaaaaaaaaa merged into trunk 111111111111 siblings unknown — (fail|pass): /, s.description);
|
||||||
|
assert.ok(s.description.length <= 140);
|
||||||
|
assert.equal(namesSiblings(s.description), false, "it read no siblings, so the merge tool asks for none");
|
||||||
|
}
|
||||||
|
assert.match(commentFor({ ...unknown, gate: { verdict: "fail", summary: "no" } })!, /Siblings could not be learned: mesh-delivery did not answer/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a verdict without siblings is said exactly as before (an older controller)", () => {
|
||||||
|
const { siblings, judge_from, ...old } = verdict;
|
||||||
|
const [gate] = statusesFor(old);
|
||||||
|
assert.equal(namesSiblings(gate.description), false);
|
||||||
|
assert.match(gate.description, /^head aaaaaaaaaaaa merged into trunk 111111111111 — pass: /);
|
||||||
|
assert.deepEqual(siblingsOf(undefined), []);
|
||||||
|
assert.deepEqual(siblingsOf([{ repository: "novox/x" }]), [], "a sibling named without its head is none");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the siblings a verdict names are kept per member, replaced by its next verdict, and removed by one that read none", () => {
|
||||||
|
const file = siblingsFile(mkdtempSync(join(tmpdir(), "gitea-siblings-")));
|
||||||
|
const record: SiblingRecord = {};
|
||||||
|
assert.equal(noteVerdict(record, "novox/mesh-controller#346", MEMBER_HEAD, []), false);
|
||||||
|
assert.equal(noteVerdict(record, "novox/mesh-controller#346", MEMBER_HEAD, siblingsOf([sibling])), true);
|
||||||
|
keepSiblings(file, record);
|
||||||
|
assert.deepEqual(readSiblings(file), record, "kept where the holder keeps its state");
|
||||||
|
assert.equal(noteVerdict(record, "novox/mesh-controller#346", MEMBER_HEAD, []), true);
|
||||||
|
assert.deepEqual(record, {});
|
||||||
|
assert.deepEqual(readSiblings(null), {});
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- a sibling's move -----------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
interface World {
|
||||||
|
host: { merged: boolean; state: string; head_sha: string; merge_commit_sha?: string };
|
||||||
|
hostBase: string;
|
||||||
|
parents: string[];
|
||||||
|
member: { head_sha: string; state: string; merged: boolean };
|
||||||
|
checks: Record<string, { state: string; description?: string }>;
|
||||||
|
written: { sha: string; context: string; state: string; description: string }[];
|
||||||
|
reads: string[];
|
||||||
|
}
|
||||||
|
const world = (): World => ({ host: { merged: false, state: "open", head_sha: HOST_HEAD }, hostBase: HOST_BASE, parents: [],
|
||||||
|
member: { head_sha: MEMBER_HEAD, state: "open", merged: false },
|
||||||
|
checks: { "mesh/merge-gate": { state: "success", description: "head aaaaaaaaaaaa merged into trunk 111111111111 beside mesh-host#114@bbbbbbbb — pass: x" },
|
||||||
|
"mesh/repo-check": { state: "success", description: "pass: y" } },
|
||||||
|
written: [], reads: [] });
|
||||||
|
const forgeOf = (w: World) => ({
|
||||||
|
async getPullRequest(owner: string, repo: string, n: number) {
|
||||||
|
w.reads.push(`${owner}/${repo}#${n}`);
|
||||||
|
return repo === "mesh-host" ? { number: n, title: "", html_url: "", ...w.host } : { number: n, title: "", html_url: "", merged: false, ...w.member };
|
||||||
|
},
|
||||||
|
async branchHead(_o: string, _r: string, _b: string) { return w.hostBase; },
|
||||||
|
async commitParents() { return w.parents; },
|
||||||
|
});
|
||||||
|
const gatesOf = (w: World) => ({
|
||||||
|
async outOfDate(_o: string, _r: string, sha: string, decide: (c: any) => any[] | Promise<any[]>) {
|
||||||
|
const put = await decide(w.checks);
|
||||||
|
for (const s of put) { w.written.push({ sha, ...s }); w.checks[s.context] = { state: s.state, description: s.description }; }
|
||||||
|
return put;
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const recordOf = (): SiblingRecord => ({ "novox/mesh-controller#346": { head: MEMBER_HEAD, siblings: siblingsOf([sibling]) } });
|
||||||
|
const weigh = (w: World, record: SiblingRecord, repository = "novox/mesh-host", number?: number) =>
|
||||||
|
weighSiblings(forgeOf(w) as any, gatesOf(w) as any, record, repository, number);
|
||||||
|
|
||||||
|
test("a sibling's new head puts both statuses of the member back to pending, with the reason, and runs nothing", async () => {
|
||||||
|
const w = world();
|
||||||
|
w.host.head_sha = HOST_NEW;
|
||||||
|
await weigh(w, recordOf(), "novox/mesh-host", 114);
|
||||||
|
assert.deepEqual(w.written.map((s) => [s.sha, s.context, s.state]),
|
||||||
|
[[MEMBER_HEAD, "mesh/merge-gate", "pending"], [MEMBER_HEAD, "mesh/repo-check", "pending"]]);
|
||||||
|
assert.equal(w.written[0].description,
|
||||||
|
"out of date: judged beside mesh-host#114 at bbbbbbbbbbbb, it is now at dddddddddddd; judged again on a recheck or when it is merged");
|
||||||
|
assert.ok(siblingOutOfDate(w.written[0].description));
|
||||||
|
// Already pending (out of date, or a check on its way): not written again.
|
||||||
|
w.written.length = 0;
|
||||||
|
await weigh(w, recordOf(), "novox/mesh-host", 114);
|
||||||
|
assert.deepEqual(w.written, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a sibling closed unmerged puts the member's verdict out of date", async () => {
|
||||||
|
const w = world();
|
||||||
|
w.host.state = "closed";
|
||||||
|
await weigh(w, recordOf(), "novox/mesh-host", 114);
|
||||||
|
assert.equal(w.written.length, 2);
|
||||||
|
assert.match(w.written[0].description, /^out of date: judged beside mesh-host#114 was closed unmerged/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a move of the sibling's base puts it out of date, naming both commits", async () => {
|
||||||
|
const w = world();
|
||||||
|
w.hostBase = HOST_MOVED;
|
||||||
|
await weigh(w, recordOf());
|
||||||
|
assert.equal(w.written.length, 2);
|
||||||
|
assert.match(w.written[0].description, /mesh-host#114 merged onto cccccccccccc, its main is now eeeeeeeeeeee/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the sibling's own merge of the head read onto the base read does not, in either order of the two merges", async () => {
|
||||||
|
// The sibling merges first (a merge commit of the base commit read and the head read).
|
||||||
|
const w = world();
|
||||||
|
Object.assign(w.host, { merged: true, state: "closed", merge_commit_sha: MERGE });
|
||||||
|
w.hostBase = MERGE;
|
||||||
|
w.parents = [HOST_BASE, HOST_HEAD];
|
||||||
|
const record = recordOf();
|
||||||
|
assert.equal(await weigh(w, record), true, "the record changes: the sibling is marked merged");
|
||||||
|
assert.deepEqual(w.written, []);
|
||||||
|
assert.equal(record["novox/mesh-controller#346"].siblings[0].merged, true);
|
||||||
|
// A move of that base afterwards is no sibling's move: the sibling is merged.
|
||||||
|
w.hostBase = HOST_MOVED;
|
||||||
|
await weigh(w, record);
|
||||||
|
assert.deepEqual(w.written, []);
|
||||||
|
|
||||||
|
// The member merges first: its own repository moves, which no member's verdict names as a sibling.
|
||||||
|
const m = world();
|
||||||
|
m.member.merged = true;
|
||||||
|
await weigh(m, recordOf(), "novox/mesh-controller");
|
||||||
|
assert.deepEqual(m.written, []);
|
||||||
|
// And the sibling then merges: still not a move.
|
||||||
|
Object.assign(m.host, { merged: true, state: "closed", merge_commit_sha: MERGE });
|
||||||
|
m.parents = [HOST_BASE, HOST_HEAD];
|
||||||
|
await weigh(m, recordOf());
|
||||||
|
assert.deepEqual(m.written, []);
|
||||||
|
|
||||||
|
// A squash is one parent: the base commit read.
|
||||||
|
const q = world();
|
||||||
|
Object.assign(q.host, { merged: true, state: "closed", merge_commit_sha: MERGE });
|
||||||
|
q.parents = [HOST_BASE];
|
||||||
|
await weigh(q, recordOf());
|
||||||
|
assert.deepEqual(q.written, []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a merge of another head, or onto another base, is a move", async () => {
|
||||||
|
const other = world();
|
||||||
|
Object.assign(other.host, { merged: true, state: "closed", merge_commit_sha: MERGE, head_sha: HOST_NEW });
|
||||||
|
other.parents = [HOST_BASE, HOST_NEW];
|
||||||
|
await weigh(other, recordOf());
|
||||||
|
assert.equal(other.written.length, 2);
|
||||||
|
const onto = world();
|
||||||
|
Object.assign(onto.host, { merged: true, state: "closed", merge_commit_sha: MERGE });
|
||||||
|
onto.parents = [HOST_MOVED, HOST_HEAD];
|
||||||
|
await weigh(onto, recordOf());
|
||||||
|
assert.equal(onto.written.length, 2);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("only the members whose verdict names the sibling are touched, and a member that moved on is dropped", async () => {
|
||||||
|
// Another sibling's move, or another repository's: nothing read, nothing written.
|
||||||
|
const w = world();
|
||||||
|
w.host.head_sha = HOST_NEW;
|
||||||
|
await weigh(w, recordOf(), "novox/mesh-lab");
|
||||||
|
await weigh(w, recordOf(), "novox/mesh-host", 999);
|
||||||
|
assert.deepEqual(w.written, []);
|
||||||
|
// A verdict that is none (a gate not yet judged) is left alone.
|
||||||
|
const none = world();
|
||||||
|
none.host.head_sha = HOST_NEW;
|
||||||
|
none.checks = {};
|
||||||
|
await weigh(none, recordOf());
|
||||||
|
assert.deepEqual(none.written, []);
|
||||||
|
// The member has a new head: its old verdict is no verdict; the record is dropped, nothing written to the old head.
|
||||||
|
const moved = world();
|
||||||
|
moved.host.head_sha = HOST_NEW;
|
||||||
|
moved.member.head_sha = h("9");
|
||||||
|
const record = recordOf();
|
||||||
|
assert.equal(await weigh(moved, record), true);
|
||||||
|
assert.deepEqual(moved.written, []);
|
||||||
|
assert.deepEqual(record, {});
|
||||||
|
// A repository without a repo-check gets no pending one a verdict would never set.
|
||||||
|
assert.deepEqual(siblingStatuses({ "mesh/merge-gate": { state: "success" } }, "x").map((s) => s.context), ["mesh/merge-gate"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a trunk that moved tells the siblings' weighing; one first seen does not", async () => {
|
||||||
|
const told: string[] = [];
|
||||||
|
const forge = { listOpenPulls: async () => [{ number: 1, title: "", state: "open", merged: false, head_sha: h("2"), base: "main", html_url: "" }],
|
||||||
|
branchHead: async () => h("3") };
|
||||||
|
const gates = { outOfDate: async () => [] };
|
||||||
|
const trunks: Record<string, string> = {};
|
||||||
|
await weighTrunks(forge as any, gates as any, trunks, "novox", "mesh-host", async (b) => { told.push(b); });
|
||||||
|
assert.deepEqual(told, [], "first seen");
|
||||||
|
trunks["novox/mesh-host:main"] = h("4");
|
||||||
|
await weighTrunks(forge as any, gates as any, trunks, "novox", "mesh-host", async (b) => { told.push(b); });
|
||||||
|
assert.deepEqual(told, ["main"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
// --- the merge rule -------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
test("the group rule names every missing thing: the group status at each head read, and each sibling head's review", async () => {
|
||||||
|
const self = { owner: "novox", repo: "mesh-controller", number: 346, head: MEMBER_HEAD };
|
||||||
|
const statuses: Record<string, string> = {};
|
||||||
|
const approved = new Set<string>();
|
||||||
|
const forge = {
|
||||||
|
headChecks: async (_o: string, r: string, sha: string) => (statuses[`${r}@${sha}`] ? { "mesh/delivery-group": { state: statuses[`${r}@${sha}`] } } : {}),
|
||||||
|
approvedAt: async (_o: string, r: string, n: number, head: string) => approved.has(`${r}#${n}@${head}`),
|
||||||
|
};
|
||||||
|
const siblings = siblingsOf([sibling]);
|
||||||
|
const none = await groupRefusal(forge, self, siblings);
|
||||||
|
assert.match(none!, /mesh\/delivery-group is missing at mesh-controller#346@aaaaaaaaaaaa; mesh\/delivery-group is missing at mesh-host#114@bbbbbbbbbbbb; mesh-host#114@bbbbbbbbbbbb carries no approving review at that head/);
|
||||||
|
statuses[`mesh-controller@${MEMBER_HEAD}`] = "success";
|
||||||
|
statuses[`mesh-host@${HOST_HEAD}`] = "pending";
|
||||||
|
assert.match((await groupRefusal(forge, self, siblings))!, /^mesh\/delivery-group is pending at mesh-host#114@bbbbbbbbbbbb; mesh-host#114@bbbbbbbbbbbb carries no approving review/);
|
||||||
|
statuses[`mesh-host@${HOST_HEAD}`] = "success";
|
||||||
|
assert.match((await groupRefusal(forge, self, siblings))!, /^mesh-host#114@bbbbbbbbbbbb carries no approving review at that head$/);
|
||||||
|
approved.add(`mesh-host#114@${HOST_HEAD}`);
|
||||||
|
assert.equal(await groupRefusal(forge, self, siblings), undefined);
|
||||||
|
// A read that fails is named, never read as a pass.
|
||||||
|
const broken = { ...forge, approvedAt: async () => { throw new Error("the forge is away"); } };
|
||||||
|
assert.match((await groupRefusal(broken, self, siblings))!, /could not be read \(the forge is away\)/);
|
||||||
|
assert.equal(besideWords(siblings), "beside mesh-host#114@bbbbbbbb");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a merged or closed member's entry is dropped, so a sibling's later push never re-pends its merged head", async () => {
|
||||||
|
// mesh-host#114 judged beside mesh-controller#346; #114 then merged; a push to #346 follows.
|
||||||
|
const w = world();
|
||||||
|
w.host.merged = true; // the member (mesh-host#114) is merged
|
||||||
|
w.member.head_sha = h("9"); // the sibling (mesh-controller#346) has a new head
|
||||||
|
const record: SiblingRecord = { "novox/mesh-host#114": { head: HOST_HEAD, siblings: siblingsOf([
|
||||||
|
{ repository: "novox/mesh-controller", number: 346, head: MEMBER_HEAD, base: "main", base_commit: h("2") }]) } };
|
||||||
|
assert.equal(await weigh(w, record, "novox/mesh-controller", 346), true);
|
||||||
|
assert.deepEqual(w.written, [], "the merged head is not put back to pending");
|
||||||
|
assert.deepEqual(record, {}, "and its entry is gone");
|
||||||
|
// On its own close or merge the member's entry goes too, whatever the siblings do.
|
||||||
|
const own = world();
|
||||||
|
own.host.state = "closed";
|
||||||
|
const kept = recordOf();
|
||||||
|
const mine: SiblingRecord = { "novox/mesh-host#114": { head: HOST_HEAD, siblings: kept["novox/mesh-controller#346"].siblings } };
|
||||||
|
assert.equal(await weigh(own, mine, "novox/mesh-host", 114), true);
|
||||||
|
assert.deepEqual(mine, {});
|
||||||
|
assert.deepEqual(own.written, []);
|
||||||
|
});
|
||||||
@@ -10,12 +10,12 @@
|
|||||||
import { test, after } from "node:test";
|
import { test, after } from "node:test";
|
||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
|
|
||||||
import { collectTools } from "@novox/mesh-sdk/tools";
|
import { collectTools } from "@novox/mesh-sdk/tools";
|
||||||
import { AdminRefused, MINT_RETRY_AFTER_MS, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
|
import { AdminRefused, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
|
||||||
import { GiteaClient } from "../dist/client.js";
|
import { GiteaClient } from "../dist/client.js";
|
||||||
import { getGiteaTools } from "../dist/tools/index.js";
|
import { getGiteaTools } from "../dist/tools/index.js";
|
||||||
|
|
||||||
@@ -29,9 +29,12 @@ interface Forge {
|
|||||||
mints: number;
|
mints: number;
|
||||||
lastScopes: string[] | null;
|
lastScopes: string[] | null;
|
||||||
tokens: Map<string, string>;
|
tokens: Map<string, string>;
|
||||||
/** Tokens beyond the one per name `tokens` holds, by id: what two mints racing leave (novox/hq issue 739). */
|
/** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */
|
||||||
twins: Map<number, { name: string; sha1: string }>;
|
twins: { id: number; name: string; sha1: string }[];
|
||||||
tokenCalls: number;
|
/** How each DELETE named its token: "id" or "name". */
|
||||||
|
deletedBy: string[];
|
||||||
|
/** Plant a token by name, beside any other of the same name, as a raced mint would. */
|
||||||
|
plant(name: string): void;
|
||||||
scopesOf: Map<string, string[]>;
|
scopesOf: Map<string, string[]>;
|
||||||
admins: Map<string, string>;
|
admins: Map<string, string>;
|
||||||
pullState: string;
|
pullState: string;
|
||||||
@@ -42,12 +45,14 @@ interface Forge {
|
|||||||
|
|
||||||
function fakeForge(): Promise<Forge> {
|
function fakeForge(): Promise<Forge> {
|
||||||
const forge = {
|
const forge = {
|
||||||
|
mintedFor: "",
|
||||||
mints: 0,
|
mints: 0,
|
||||||
lastScopes: null as string[] | null,
|
lastScopes: null as string[] | null,
|
||||||
tokens: new Map<string, string>(), // name -> value
|
tokens: new Map<string, string>(), // name -> value
|
||||||
twins: new Map<number, { name: string; sha1: string }>(),
|
twins: [] as { id: number; name: string; sha1: string }[],
|
||||||
idOf: new Map<string, number>(), // value -> id
|
deletedBy: [] as string[],
|
||||||
tokenCalls: 0,
|
ids: new Map<string, number>(), // value -> id, as gitea numbers its tokens
|
||||||
|
nextId: 0,
|
||||||
scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
|
scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
|
||||||
admins: new Map([[ADMIN, PASSWORD]]),
|
admins: new Map([[ADMIN, PASSWORD]]),
|
||||||
pullState: "open",
|
pullState: "open",
|
||||||
@@ -79,41 +84,51 @@ function fakeForge(): Promise<Forge> {
|
|||||||
const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
|
const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
|
||||||
if (tokens) {
|
if (tokens) {
|
||||||
const user = basic(req);
|
const user = basic(req);
|
||||||
forge.tokenCalls++;
|
// gitea's reqSelfOrAdmin: the account itself, or the administrator for another (the tools' account, ADR 0365).
|
||||||
if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" });
|
if (user === null || (user !== decodeURIComponent(tokens[1]) && user !== ADMIN)) return json(res, 401, { message: "auth required" });
|
||||||
const named = (name: string): number =>
|
// This fake keeps one account's tokens: the tools' account's when the module mints for it (ADR 0365), and the
|
||||||
(forge.tokens.has(name) ? 1 : 0) + [...forge.twins.values()].filter((t) => t.name === name).length;
|
// admin's own list is then empty — nothing of it is retired.
|
||||||
|
if (decodeURIComponent(tokens[1]) === ADMIN && forge.mintedFor === "mesh-tools") {
|
||||||
|
return req.method === "GET" ? json(res, 200, []) : json(res, 404, { message: "token not found" });
|
||||||
|
}
|
||||||
|
if (req.method === "POST") forge.mintedFor = decodeURIComponent(tokens[1]);
|
||||||
if (req.method === "POST") {
|
if (req.method === "POST") {
|
||||||
const { name, scopes } = await body(req);
|
const { name, scopes } = await body(req);
|
||||||
if (named(name) > 0) return json(res, 400, { message: "token name has already been used" });
|
if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" });
|
||||||
forge.mints++;
|
forge.mints++;
|
||||||
forge.lastScopes = scopes;
|
forge.lastScopes = scopes;
|
||||||
const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
|
const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
|
||||||
forge.tokens.set(name, sha1);
|
forge.tokens.set(name, sha1);
|
||||||
forge.idOf.set(sha1, forge.mints);
|
|
||||||
forge.scopesOf.set(sha1, scopes);
|
forge.scopesOf.set(sha1, scopes);
|
||||||
return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) });
|
forge.ids.set(sha1, ++forge.nextId);
|
||||||
|
return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) });
|
||||||
}
|
}
|
||||||
|
// gitea's list: every token of the account, a page at a time (at most three to a page here, so a
|
||||||
|
// client that reads only the first page misses some), the values never shown.
|
||||||
|
const listed = (): { id: number; name: string; sha1: string }[] => [
|
||||||
|
...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })),
|
||||||
|
...forge.twins,
|
||||||
|
];
|
||||||
if (req.method === "GET" && !tokens[2]) {
|
if (req.method === "GET" && !tokens[2]) {
|
||||||
const page = Number(url.searchParams.get("page") ?? "1");
|
const page = Math.max(1, Number(url.searchParams.get("page") ?? "1"));
|
||||||
const all = [...[...forge.tokens].map(([name, v]) => ({ id: forge.idOf.get(v) ?? 0, name })),
|
const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3")));
|
||||||
...[...forge.twins].map(([id, t]) => ({ id, name: t.name }))];
|
const all = listed();
|
||||||
return json(res, 200, page === 1 ? all : []);
|
res.setHeader("X-Total-Count", String(all.length));
|
||||||
|
return json(res, 200, all.slice((page - 1) * limit, page * limit)
|
||||||
|
.map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] })));
|
||||||
}
|
}
|
||||||
if (req.method === "DELETE" && tokens[2]) {
|
if (req.method === "DELETE" && tokens[2]) {
|
||||||
|
// gitea: "identified by ID and if not available by name" — and a name held twice is refused.
|
||||||
const key = decodeURIComponent(tokens[2]);
|
const key = decodeURIComponent(tokens[2]);
|
||||||
// gitea: "identified by ID and if not available by name"
|
const byId = /^\d+$/.test(key);
|
||||||
if (/^\d+$/.test(key)) {
|
const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key));
|
||||||
const id = Number(key);
|
if (matches.length === 0) return json(res, 404, { message: "token not found" });
|
||||||
if (forge.twins.delete(id)) return json(res, 204, null);
|
if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
|
||||||
const hit = [...forge.tokens].find(([, v]) => forge.idOf.get(v) === id);
|
const t = matches[0];
|
||||||
if (!hit) return json(res, 404, { message: "token not found" });
|
forge.deletedBy.push(byId ? "id" : "name");
|
||||||
forge.tokens.delete(hit[0]);
|
const twin = forge.twins.findIndex((w) => w.id === t.id);
|
||||||
return json(res, 204, null);
|
if (twin >= 0) forge.twins.splice(twin, 1);
|
||||||
}
|
else forge.tokens.delete(t.name);
|
||||||
if (named(key) > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
|
|
||||||
if (!forge.tokens.has(key)) return json(res, 404, { message: "token not found" });
|
|
||||||
forge.tokens.delete(key);
|
|
||||||
return json(res, 204, null);
|
return json(res, 204, null);
|
||||||
}
|
}
|
||||||
return json(res, 405, { message: "method not allowed" });
|
return json(res, 405, { message: "method not allowed" });
|
||||||
@@ -179,6 +194,17 @@ function fakeForge(): Promise<Forge> {
|
|||||||
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
|
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
// What setting up the tools' account asks with the admin login (tools-account.ts, ADR 0365): answered plainly.
|
||||||
|
if (basic(req) === ADMIN) {
|
||||||
|
const p = url.pathname;
|
||||||
|
if (req.method === "GET" && p === "/api/v1/user") return json(res, 200, { login: ADMIN, is_admin: true });
|
||||||
|
if (req.method === "GET" && p === "/api/v1/users/mesh-tools") return json(res, 200, { login: "mesh-tools", is_admin: false });
|
||||||
|
if (req.method === "POST" && p === "/api/v1/admin/users") return json(res, 422, { message: "user already exists" });
|
||||||
|
if (req.method === "PATCH" && p === "/api/v1/admin/users/mesh-tools") return json(res, 200, { login: "mesh-tools" });
|
||||||
|
if (req.method === "GET" && p === "/api/v1/orgs/novox/teams") return json(res, 200, [{ id: 7, name: "mesh-tools" }]);
|
||||||
|
if (req.method === "PATCH" && p === "/api/v1/teams/7") return json(res, 200, { id: 7 });
|
||||||
|
if (req.method === "PUT" && p === "/api/v1/teams/7/members/mesh-tools") return json(res, 204, null);
|
||||||
|
}
|
||||||
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
|
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
|
||||||
if (adminUser && req.method === "PATCH") {
|
if (adminUser && req.method === "PATCH") {
|
||||||
const h = req.headers.authorization ?? "";
|
const h = req.headers.authorization ?? "";
|
||||||
@@ -206,9 +232,15 @@ function fakeForge(): Promise<Forge> {
|
|||||||
get pullState() { return forge.pullState; },
|
get pullState() { return forge.pullState; },
|
||||||
get pullTitle() { return forge.pullTitle; },
|
get pullTitle() { return forge.pullTitle; },
|
||||||
get branchDeleted() { return forge.branchDeleted; },
|
get branchDeleted() { return forge.branchDeleted; },
|
||||||
get tokenCalls() { return forge.tokenCalls; },
|
|
||||||
tokens: forge.tokens,
|
tokens: forge.tokens,
|
||||||
twins: forge.twins,
|
twins: forge.twins,
|
||||||
|
deletedBy: forge.deletedBy,
|
||||||
|
plant: (name: string) => {
|
||||||
|
const sha1 = `planted-${++forge.nextId}`;
|
||||||
|
forge.ids.set(sha1, forge.nextId);
|
||||||
|
if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 });
|
||||||
|
else forge.tokens.set(name, sha1);
|
||||||
|
},
|
||||||
scopesOf: forge.scopesOf,
|
scopesOf: forge.scopesOf,
|
||||||
admins: forge.admins,
|
admins: forge.admins,
|
||||||
close: () => new Promise((r) => server.close(() => r())),
|
close: () => new Promise((r) => server.close(() => r())),
|
||||||
@@ -237,9 +269,8 @@ async function delivered(forge: Forge): Promise<{ env: NodeJS.ProcessEnv; file:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** A client as a fresh process would build it: a new source over the kept file, its log captured. */
|
/** A client as a fresh process would build it: a new source over the kept file, its log captured. */
|
||||||
function minted(env: NodeJS.ProcessEnv, logs: string[], now?: () => number): GiteaClient {
|
function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient {
|
||||||
const source = new MintedToken({
|
const source = new MintedToken({
|
||||||
now,
|
|
||||||
url: env.MESH_GITEA_URL!,
|
url: env.MESH_GITEA_URL!,
|
||||||
admin: env.MESH_GITEA_ADMIN_USER!,
|
admin: env.MESH_GITEA_ADMIN_USER!,
|
||||||
passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
||||||
@@ -354,44 +385,6 @@ test("the kept file is gone but the forge still holds a token by that name: repl
|
|||||||
assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
|
assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
|
||||||
});
|
});
|
||||||
|
|
||||||
test("two tokens carry the name (two mints raced): each is dropped by its id and one is minted, not a 422 for good (issue 739)", async () => {
|
|
||||||
const { env, logs } = await delivered(forge);
|
|
||||||
await minted(env, logs).listRepos();
|
|
||||||
// What two processes minting at once leave: a second token by the same name, whose value nobody kept.
|
|
||||||
// Two tokens by the name, neither of them the one kept.
|
|
||||||
forge.tokens.clear();
|
|
||||||
forge.twins.set(9001, { name: "mesh-tools", sha1: "raced-first" });
|
|
||||||
forge.twins.set(9002, { name: "mesh-tools", sha1: "raced-second" });
|
|
||||||
|
|
||||||
const repos = await minted(env, logs).listRepos();
|
|
||||||
|
|
||||||
assert.equal(repos.length, 1);
|
|
||||||
assert.equal(forge.twins.size, 0);
|
|
||||||
assert.equal([...forge.tokens.keys()].filter((n) => n === "mesh-tools").length, 1);
|
|
||||||
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
|
|
||||||
});
|
|
||||||
|
|
||||||
test("a mint that failed is not asked of the forge again within the retry window: the same error, no basic-auth call (issue 739)", async () => {
|
|
||||||
const { env, logs } = await delivered(forge);
|
|
||||||
forge.admins.delete(ADMIN);
|
|
||||||
try {
|
|
||||||
let clock = 5_000_000;
|
|
||||||
const client = minted(env, logs, () => clock);
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
const calls = forge.tokenCalls;
|
|
||||||
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
assert.equal(forge.tokenCalls, calls);
|
|
||||||
|
|
||||||
clock += MINT_RETRY_AFTER_MS;
|
|
||||||
await assert.rejects(client.listRepos(), AdminRefused);
|
|
||||||
assert.equal(forge.tokenCalls, calls + 1);
|
|
||||||
} finally {
|
|
||||||
forge.admins.set(ADMIN, PASSWORD);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
test("concurrent first calls share one mint", async () => {
|
test("concurrent first calls share one mint", async () => {
|
||||||
const { env, logs } = await delivered(forge);
|
const { env, logs } = await delivered(forge);
|
||||||
const client = minted(env, logs);
|
const client = minted(env, logs);
|
||||||
@@ -402,12 +395,61 @@ test("concurrent first calls share one mint", async () => {
|
|||||||
assert.equal(forge.mints, before + 1);
|
assert.equal(forge.mints, before + 1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => {
|
||||||
|
const { env, file, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept
|
||||||
|
forge.plant("mesh-tools");
|
||||||
|
forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on
|
||||||
|
forge.deletedBy.length = 0;
|
||||||
|
const before = forge.mints;
|
||||||
|
|
||||||
|
const repos = await minted(env, logs).listRepos();
|
||||||
|
|
||||||
|
assert.equal(repos.length, 1);
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
assert.deepEqual(forge.deletedBy, ["id", "id"]);
|
||||||
|
assert.equal(forge.twins.length, 0);
|
||||||
|
assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]);
|
||||||
|
assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n");
|
||||||
|
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => {
|
||||||
|
const { env, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
const before = forge.mints;
|
||||||
|
// Two sources over one state directory, as the tools bundle and the watcher are two processes.
|
||||||
|
const first = minted(env, logs);
|
||||||
|
const second = minted(env, logs);
|
||||||
|
|
||||||
|
await Promise.all([first.listRepos(), second.listRepos()]);
|
||||||
|
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
assert.equal(forge.twins.length, 0);
|
||||||
|
assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => {
|
||||||
|
const { env, file, logs } = await delivered(forge);
|
||||||
|
forge.tokens.clear();
|
||||||
|
forge.twins.length = 0;
|
||||||
|
await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true });
|
||||||
|
await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds
|
||||||
|
const before = forge.mints;
|
||||||
|
|
||||||
|
assert.equal((await minted(env, logs).listRepos()).length, 1);
|
||||||
|
assert.equal(forge.mints, before + 1);
|
||||||
|
await assert.rejects(stat(`${file}.lock`), /ENOENT/);
|
||||||
|
});
|
||||||
|
|
||||||
test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
|
test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
|
||||||
const { env, file, logs } = await delivered(forge);
|
const { env, file, logs } = await delivered(forge);
|
||||||
forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
|
forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
|
||||||
try {
|
try {
|
||||||
let clock = 1_000_000;
|
const client = minted(env, logs);
|
||||||
const client = minted(env, logs, () => clock);
|
|
||||||
const before = forge.mints;
|
const before = forge.mints;
|
||||||
|
|
||||||
await assert.rejects(client.listRepos(), (err: unknown) => {
|
await assert.rejects(client.listRepos(), (err: unknown) => {
|
||||||
@@ -422,9 +464,8 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal
|
|||||||
assert.equal(forge.mints, before);
|
assert.equal(forge.mints, before);
|
||||||
await assert.rejects(stat(file), /ENOENT/);
|
await assert.rejects(stat(file), /ENOENT/);
|
||||||
|
|
||||||
// The account appears (the operator created it): the next call after the retry window mints and works.
|
// The account appears (the operator created it): the very next call mints and works.
|
||||||
forge.admins.set(ADMIN, PASSWORD);
|
forge.admins.set(ADMIN, PASSWORD);
|
||||||
clock += MINT_RETRY_AFTER_MS;
|
|
||||||
assert.equal((await client.listRepos()).length, 1);
|
assert.equal((await client.listRepos()).length, 1);
|
||||||
assert.equal(forge.mints, before + 1);
|
assert.equal(forge.mints, before + 1);
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -0,0 +1,332 @@
|
|||||||
|
// The forge's tools act as an account of their own, not the site administrator (novox/hq ADR 0365 §2, issue 469):
|
||||||
|
// the token the tools and the watcher use is minted for `mesh-tools`, a plain account made by the module with the
|
||||||
|
// administrator's login and given write on the mesh's repositories through one team; the administrator's own
|
||||||
|
// token by that name goes; a token kept from before, the administrator's, is minted again for the account.
|
||||||
|
//
|
||||||
|
// The forge is a fake that keeps accounts, their tokens and the organisation's teams apart, and answers as gitea
|
||||||
|
// does: an administrator may mint for another account (reqSelfOrAdmin), anyone else only for itself; a protection
|
||||||
|
// edit needs the repository's administration, which neither a plain account nor its write team holds.
|
||||||
|
|
||||||
|
import { test, after } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||||
|
import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
import { MintedToken } from "../dist/token.js";
|
||||||
|
import { GiteaClient } from "../dist/client.js";
|
||||||
|
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js";
|
||||||
|
import { repoCheckFacts } from "../dist/pulls.js";
|
||||||
|
|
||||||
|
const ADMIN = "mesh-admin";
|
||||||
|
const PASSWORD = "the-vault-minted-this";
|
||||||
|
|
||||||
|
interface Account { admin: boolean; password: string; active: boolean }
|
||||||
|
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
|
||||||
|
interface Team { id: number; name: string; units: any; members: Set<string> }
|
||||||
|
|
||||||
|
function fakeForge() {
|
||||||
|
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
|
||||||
|
const tokens: Token[] = [];
|
||||||
|
const teams: Team[] = [];
|
||||||
|
const calls: string[] = [];
|
||||||
|
// What the forge does wrong when a test says so: a token mint or delete it fails, a team another process made
|
||||||
|
// between this one's look and its create (gitea answers the create 422).
|
||||||
|
const faults = { mint: false, deleteToken: false, teamRace: false };
|
||||||
|
let next = 0;
|
||||||
|
const json = (res: ServerResponse, status: number, body: unknown): void => {
|
||||||
|
res.writeHead(status, { "Content-Type": "application/json" });
|
||||||
|
res.end(body === null ? "" : JSON.stringify(body));
|
||||||
|
};
|
||||||
|
const read = (req: IncomingMessage): Promise<any> =>
|
||||||
|
new Promise((resolve) => {
|
||||||
|
let t = "";
|
||||||
|
req.on("data", (c) => (t += c));
|
||||||
|
req.on("end", () => resolve(t ? JSON.parse(t) : null));
|
||||||
|
});
|
||||||
|
const who = (req: IncomingMessage): string | null => {
|
||||||
|
const h = req.headers.authorization ?? "";
|
||||||
|
if (h.startsWith("Basic ")) {
|
||||||
|
const [u, p] = Buffer.from(h.slice(6), "base64").toString().split(":");
|
||||||
|
return users.get(u)?.password === p ? u : null;
|
||||||
|
}
|
||||||
|
if (h.startsWith("token ")) return tokens.find((t) => t.sha1 === h.slice(6))?.user ?? null;
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
|
||||||
|
|
||||||
|
const server = createServer(async (req, res) => {
|
||||||
|
const url = new URL(req.url ?? "/", "http://fake");
|
||||||
|
const p = url.pathname.replace(/^\/api\/v1/, "");
|
||||||
|
const me = who(req);
|
||||||
|
calls.push(`${req.method} ${p} as ${me}`);
|
||||||
|
if (me === null) return json(res, 401, { message: "unauthorized" });
|
||||||
|
|
||||||
|
let m = p.match(/^\/users\/([^/]+)\/tokens(?:\/(\d+))?$/);
|
||||||
|
if (m) {
|
||||||
|
const owner = decodeURIComponent(m[1]);
|
||||||
|
if (!(req.headers.authorization ?? "").startsWith("Basic ")) return json(res, 401, { message: "basic auth required" });
|
||||||
|
if (owner !== me && !isAdmin(me)) return json(res, 403, { message: "reqSelfOrAdmin" });
|
||||||
|
if (!users.has(owner)) return json(res, 404, { message: "user does not exist" });
|
||||||
|
if (req.method === "POST") {
|
||||||
|
if (faults.mint) return json(res, 500, { message: "the forge is not well" });
|
||||||
|
const { name, scopes } = await read(req);
|
||||||
|
if (tokens.some((t) => t.user === owner && t.name === name)) return json(res, 400, { message: "token name has already been used" });
|
||||||
|
const t = { id: ++next, name, sha1: `sha-${owner}-${next}`, user: owner, scopes };
|
||||||
|
tokens.push(t);
|
||||||
|
return json(res, 201, { id: t.id, name, sha1: t.sha1, scopes });
|
||||||
|
}
|
||||||
|
if (req.method === "GET") {
|
||||||
|
const page = Number(url.searchParams.get("page") ?? "1");
|
||||||
|
const mine = tokens.filter((t) => t.user === owner);
|
||||||
|
return json(res, 200, page === 1 ? mine.map((t) => ({ id: t.id, name: t.name, sha1: "" })) : []);
|
||||||
|
}
|
||||||
|
if (req.method === "DELETE" && m[2]) {
|
||||||
|
if (faults.deleteToken && owner === ADMIN) return json(res, 500, { message: "the forge is not well" });
|
||||||
|
const i = tokens.findIndex((t) => t.user === owner && t.id === Number(m![2]));
|
||||||
|
if (i < 0) return json(res, 404, { message: "not found" });
|
||||||
|
tokens.splice(i, 1);
|
||||||
|
return json(res, 204, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) });
|
||||||
|
m = p.match(/^\/users\/([^/]+)$/);
|
||||||
|
if (m && req.method === "GET") {
|
||||||
|
const u = users.get(decodeURIComponent(m[1]));
|
||||||
|
if (!u) return json(res, 404, { message: "user does not exist" });
|
||||||
|
return json(res, 200, { login: m[1], is_admin: u.admin, active: u.active, prohibit_login: false });
|
||||||
|
}
|
||||||
|
if (p.startsWith("/admin/") && !isAdmin(me)) return json(res, 403, { message: "must be a site administrator" });
|
||||||
|
if (req.method === "POST" && p === "/admin/users") {
|
||||||
|
const b = await read(req);
|
||||||
|
if (users.has(b.username)) return json(res, 422, { message: "user already exists" });
|
||||||
|
if (!b.password || b.password.length < 8) return json(res, 422, { message: "password is too short" });
|
||||||
|
users.set(b.username, { admin: b.admin === true, password: b.password, active: true });
|
||||||
|
return json(res, 201, { login: b.username, is_admin: b.admin === true });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/admin\/users\/([^/]+)$/);
|
||||||
|
if (m && req.method === "PATCH") {
|
||||||
|
const u = users.get(decodeURIComponent(m[1]));
|
||||||
|
if (!u) return json(res, 404, { message: "user does not exist" });
|
||||||
|
const b = await read(req);
|
||||||
|
if (b.admin !== undefined) u.admin = b.admin;
|
||||||
|
if (b.active !== undefined) u.active = b.active;
|
||||||
|
return json(res, 200, { login: m[1], is_admin: u.admin });
|
||||||
|
}
|
||||||
|
if (p === "/orgs/novox/teams") {
|
||||||
|
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
|
||||||
|
if (req.method === "GET") return json(res, 200, teams.map((t) => ({ id: t.id, name: t.name })));
|
||||||
|
const b = await read(req);
|
||||||
|
const t = { id: ++next, name: b.name, units: b, members: new Set<string>() };
|
||||||
|
teams.push(t);
|
||||||
|
if (faults.teamRace) return json(res, 422, { message: "team already exists" });
|
||||||
|
return json(res, 201, { id: t.id, name: t.name });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/teams\/(\d+)(?:\/members\/([^/]+))?$/);
|
||||||
|
if (m) {
|
||||||
|
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
|
||||||
|
const t = teams.find((x) => x.id === Number(m![1]));
|
||||||
|
if (!t) return json(res, 404, { message: "team does not exist" });
|
||||||
|
if (req.method === "PATCH") { t.units = await read(req); return json(res, 200, { id: t.id }); }
|
||||||
|
if (req.method === "PUT" && m[2]) { t.members.add(decodeURIComponent(m[2])); return json(res, 204, null); }
|
||||||
|
}
|
||||||
|
// Repository administration: a site administrator, or a team member whose team administers the repository.
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/branches\/(.+)$/);
|
||||||
|
if (m && req.method === "GET") {
|
||||||
|
if (m[2] === "main") return json(res, 200, { name: "main", protected: true, effective_branch_protection_name: "main",
|
||||||
|
enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], user_can_push: false });
|
||||||
|
if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false });
|
||||||
|
return json(res, 404, { message: "branch does not exist" });
|
||||||
|
}
|
||||||
|
// gitea's reqAdmin on the whole group: reading the rules is administration too.
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/.*)?$/);
|
||||||
|
if (m) {
|
||||||
|
const admins = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.permission === "admin");
|
||||||
|
if (!admins) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
|
||||||
|
return json(res, 201, { rule_name: "main" });
|
||||||
|
}
|
||||||
|
m = p.match(/^\/repos\/novox\/([^/]+)\/statuses\/([0-9a-f]+)$/);
|
||||||
|
if (m && req.method === "POST") {
|
||||||
|
const writes = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.code"] === "write");
|
||||||
|
if (!writes) return json(res, 403, { message: "write on code required" });
|
||||||
|
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
|
||||||
|
}
|
||||||
|
if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] });
|
||||||
|
return json(res, 404, { message: "no such route in the fake" });
|
||||||
|
});
|
||||||
|
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[];
|
||||||
|
faults: typeof faults; close(): Promise<void> }>((resolve) => {
|
||||||
|
server.listen(0, "127.0.0.1", () => {
|
||||||
|
const { port } = server.address() as { port: number };
|
||||||
|
resolve({ url: `http://127.0.0.1:${port}`, users, tokens, teams, calls, faults,
|
||||||
|
close: () => new Promise((r) => server.close(() => r())) });
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const forge = await fakeForge();
|
||||||
|
after(() => forge.close());
|
||||||
|
|
||||||
|
async function environment(): Promise<NodeJS.ProcessEnv & { MESH_GITEA_STATE_DIR: string }> {
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-"));
|
||||||
|
const passwordFile = join(dir, "admin.secret");
|
||||||
|
await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 });
|
||||||
|
return {
|
||||||
|
MESH_GITEA_URL: forge.url,
|
||||||
|
MESH_GITEA_ADMIN_USER: ADMIN,
|
||||||
|
MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile,
|
||||||
|
MESH_GITEA_STATE_DIR: join(dir, "state"),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A source as a fresh process builds it from the manifest's environment (fromEnv shares one per file). */
|
||||||
|
function fresh(env: NodeJS.ProcessEnv): GiteaClient {
|
||||||
|
return GiteaClient.fromEnv(env);
|
||||||
|
}
|
||||||
|
|
||||||
|
test("the tools' token is minted for an account of their own, which is no administrator and writes through its team", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
|
||||||
|
const account = forge.users.get(TOOLS_ACCOUNT);
|
||||||
|
assert.ok(account, "the module made the tools' account");
|
||||||
|
assert.equal(account!.admin, false, "the tools' account is no administrator");
|
||||||
|
const held = forge.tokens.filter((t) => t.name === "mesh-tools");
|
||||||
|
assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's");
|
||||||
|
assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n");
|
||||||
|
|
||||||
|
const team = forge.teams.find((t) => t.name === "mesh-tools");
|
||||||
|
assert.ok(team, "the team is made in the mesh's organisation");
|
||||||
|
assert.ok(team!.members.has(TOOLS_ACCOUNT));
|
||||||
|
assert.equal(team!.units.permission, "write");
|
||||||
|
assert.equal(team!.units.includes_all_repositories, true);
|
||||||
|
assert.equal(team!.units.can_create_org_repo, false, "a repository's maker would administer it");
|
||||||
|
assert.deepEqual(team!.units.units_map, TOOLS_TEAM_UNITS.units_map);
|
||||||
|
assert.ok(!Object.values(team!.units.units_map).includes("admin"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("with the tools' token, the forge writes a status and refuses a protection edit — the forge itself, not a guard", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const client = fresh(env);
|
||||||
|
await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any);
|
||||||
|
await assert.rejects(
|
||||||
|
client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }),
|
||||||
|
/403/,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("what a base's protection asks is read from the branch, which the tools' account may read, not from the rules' list it may not", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const client = fresh(env);
|
||||||
|
await assert.rejects(client.branchProtections("novox", "hq"), /403/);
|
||||||
|
const rule = await client.appliedProtection("novox", "hq", "main");
|
||||||
|
assert.deepEqual(rule?.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]);
|
||||||
|
assert.equal(rule?.enable_status_check, true);
|
||||||
|
assert.equal(rule?.rule_name, "main");
|
||||||
|
assert.equal(await client.appliedProtection("novox", "hq", "feat/unguarded"), null);
|
||||||
|
assert.equal(await client.appliedProtection("novox", "hq", "no/such-branch"), null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the administrator's token from before goes: a kept admin token is minted again for the tools' account, and the forge drops the old one", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
// What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it.
|
||||||
|
const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true });
|
||||||
|
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 });
|
||||||
|
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 });
|
||||||
|
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
|
||||||
|
assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more");
|
||||||
|
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||||
|
assert.notEqual(kept, old.sha1);
|
||||||
|
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT);
|
||||||
|
assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
forge.users.get(TOOLS_ACCOUNT)!.admin = true;
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
assert.equal(forge.users.get(TOOLS_ACCOUNT)!.admin, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the tools never act as the administrator: MESH_GITEA_TOOLS_USER naming it is refused", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
assert.throws(() => MintedToken.fromEnv(forge.url, { ...env, MESH_GITEA_TOOLS_USER: ADMIN }), /names the admin account/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the manifest hands the tools their account, and the merge-ready authors name it rather than the administrator", async () => {
|
||||||
|
const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8"));
|
||||||
|
const code = manifest.build.artifacts.find((a: any) => a.name === "code");
|
||||||
|
assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT);
|
||||||
|
const authors = String(manifest.settings["ready-authors"].default).split(/\s+/);
|
||||||
|
assert.ok(authors.includes(TOOLS_ACCOUNT));
|
||||||
|
// Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as
|
||||||
|
// their own keeps counting as ready after the delivery.
|
||||||
|
assert.ok(authors.includes(ADMIN), "marks made by the admin account before ADR 0365 keep counting");
|
||||||
|
// The Go forge bundle reads the token the tools' account holds, never the administrator's password.
|
||||||
|
const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge");
|
||||||
|
assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE);
|
||||||
|
assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token.
|
||||||
|
|
||||||
|
test("a repository check said as a warning on a head without merge-check.sh reads the base's protection from the branch", async () => {
|
||||||
|
const client = fresh(await environment());
|
||||||
|
const forgeOf = { holdsFile: async () => false, appliedProtection: client.appliedProtection.bind(client) };
|
||||||
|
const c = { owner: "novox", repo: "hq", commit: "abc123def4567890abc123def4567890abc12345", "repo-check": { verdict: "warning" } } as any;
|
||||||
|
// main requires mesh/repo-check: required, so a head that defines no check fails it for a person to see.
|
||||||
|
assert.deepEqual(await repoCheckFacts(forgeOf, c, "main"), { defined: false, required: true });
|
||||||
|
// A base nothing guards: not required, so the head's repository check passes rather than fails.
|
||||||
|
assert.deepEqual(await repoCheckFacts(forgeOf, c, "feat/unguarded"), { defined: false, required: false });
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a mint that fails leaves the administrator's old token in place: the tools are never left without one", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const old = { id: 901, name: "mesh-tools", sha1: "sha-admin-kept", user: ADMIN, scopes: ["read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
forge.faults.mint = true;
|
||||||
|
try {
|
||||||
|
await assert.rejects(fresh(env).listRepos(), /500/);
|
||||||
|
} finally {
|
||||||
|
forge.faults.mint = false;
|
||||||
|
}
|
||||||
|
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's token was deleted before a new one was kept");
|
||||||
|
forge.tokens.splice(forge.tokens.indexOf(old), 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("an old token that cannot be deleted after the mint is said, the new token kept, and deleted at the next start", async () => {
|
||||||
|
const env = await environment();
|
||||||
|
const old = { id: 902, name: "mesh-tools", sha1: "sha-admin-stuck", user: ADMIN, scopes: ["read:user"] };
|
||||||
|
forge.tokens.push(old);
|
||||||
|
forge.faults.deleteToken = true;
|
||||||
|
try {
|
||||||
|
await fresh(env).listRepos();
|
||||||
|
} finally {
|
||||||
|
forge.faults.deleteToken = false;
|
||||||
|
}
|
||||||
|
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
|
||||||
|
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works");
|
||||||
|
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1));
|
||||||
|
// The next process reuses the kept token and retires the old one it could not before.
|
||||||
|
const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
|
||||||
|
file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} });
|
||||||
|
assert.equal(await next.current(), kept);
|
||||||
|
assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a team another process made between the look and the create is found, not a failure", async () => {
|
||||||
|
forge.teams.splice(0);
|
||||||
|
forge.faults.teamRace = true;
|
||||||
|
try {
|
||||||
|
await fresh(await environment()).listRepos();
|
||||||
|
} finally {
|
||||||
|
forge.faults.teamRace = false;
|
||||||
|
}
|
||||||
|
const team = forge.teams.find((t) => t.name === "mesh-tools");
|
||||||
|
assert.ok(team?.members.has(TOOLS_ACCOUNT));
|
||||||
|
});
|
||||||
@@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () =>
|
|||||||
assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
|
assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the system webhook is made when none points at the receiver, and corrected when one does", async () => {
|
// A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook
|
||||||
const { ensureSystemHook } = await import("../webhook.ts");
|
// is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default).
|
||||||
|
// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time.
|
||||||
|
function fakeForge(start: { id: number; url: string; system: boolean }[] = [],
|
||||||
|
repoHooks: Record<string, { id: number; url: string }[]> = {}, repoCount = 0) {
|
||||||
|
const hooks = start.map((h) => ({ ...h, events: ["push"], active: true }));
|
||||||
|
const repos = [...Object.keys(repoHooks)];
|
||||||
|
for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`);
|
||||||
|
let next = 100;
|
||||||
const calls: string[] = [];
|
const calls: string[] = [];
|
||||||
let hooks: unknown[] = [];
|
const made: any[] = [];
|
||||||
|
const page = (items: any[], q: URLSearchParams) => {
|
||||||
|
const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50);
|
||||||
|
return items.slice((p - 1) * l, p * l);
|
||||||
|
};
|
||||||
const admin = { call: async (method: string, path: string, body?: any) => {
|
const admin = { call: async (method: string, path: string, body?: any) => {
|
||||||
calls.push(`${method} ${path}`);
|
calls.push(`${method} ${path}`);
|
||||||
if (method === "GET") return { status: 200, body: hooks };
|
const u = new URL(path, "http://f");
|
||||||
if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; }
|
const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/);
|
||||||
return { status: 200, body: {} };
|
if (method === "GET" && u.pathname === "/repos/search") {
|
||||||
|
return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } };
|
||||||
|
}
|
||||||
|
if (repoHook) {
|
||||||
|
const list = repoHooks[repoHook[1]] ?? [];
|
||||||
|
if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) };
|
||||||
|
const i = list.findIndex((h) => h.id === Number(repoHook[2]));
|
||||||
|
if (i < 0) return { status: 404, body: {} };
|
||||||
|
list.splice(i, 1);
|
||||||
|
return { status: 204, body: null };
|
||||||
|
}
|
||||||
|
if (method === "GET") {
|
||||||
|
const type = u.searchParams.get("type") ?? "system";
|
||||||
|
const listed = hooks.filter((h) => type === "all" || h.system === (type === "system"));
|
||||||
|
return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events,
|
||||||
|
config: { url: h.url, content_type: "json" } })) };
|
||||||
|
}
|
||||||
|
if (method === "POST") {
|
||||||
|
made.push(body);
|
||||||
|
hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events,
|
||||||
|
active: body.active });
|
||||||
|
return { status: 201, body: {} };
|
||||||
|
}
|
||||||
|
if (method === "DELETE") {
|
||||||
|
const id = Number(path.split("/").pop());
|
||||||
|
const i = hooks.findIndex((h) => h.id === id);
|
||||||
|
if (i < 0) return { status: 404, body: {} };
|
||||||
|
hooks.splice(i, 1);
|
||||||
|
return { status: 204, body: null };
|
||||||
|
}
|
||||||
|
return { status: 405, body: {} };
|
||||||
} };
|
} };
|
||||||
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created");
|
return { admin, hooks, calls, made, repoHooks };
|
||||||
hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }];
|
}
|
||||||
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected");
|
|
||||||
assert.ok(calls.includes("PATCH /admin/hooks/4"));
|
test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const forge = fakeForge();
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.equal(forge.made.length, 1);
|
||||||
|
assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository");
|
||||||
|
assert.equal(forge.made[0].config.secret, secret);
|
||||||
|
assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push"));
|
||||||
|
assert.deepEqual(forge.hooks.map((h) => h.system), [true]);
|
||||||
|
assert.equal(hook.kind, "system");
|
||||||
|
assert.equal(hook.did, "created");
|
||||||
|
assert.equal(hook.url, url);
|
||||||
|
assert.ok(hook.events.includes("pull_request"));
|
||||||
|
});
|
||||||
|
|
||||||
|
test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
// What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start.
|
||||||
|
const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]);
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] });
|
||||||
|
assert.equal(hook.did, "replaced");
|
||||||
|
assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept");
|
||||||
|
// A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never
|
||||||
|
// changes a secret).
|
||||||
|
const again = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] });
|
||||||
|
assert.equal(forge.hooks.filter((h) => h.url === url).length, 1);
|
||||||
|
assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5"));
|
||||||
|
// The new one is made before any old one is deleted, so no event falls in a gap.
|
||||||
|
const post = forge.calls.indexOf("POST /admin/hooks");
|
||||||
|
assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const url = "http://h:3047/forge-event";
|
||||||
|
// A repository made after the default webhook got its copy, which would send its events a second time; 120
|
||||||
|
// repositories, so the copies sit beyond the first page.
|
||||||
|
const repoHooks: Record<string, { id: number; url: string }[]> = {};
|
||||||
|
for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = [];
|
||||||
|
repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }];
|
||||||
|
repoHooks["novox/later"] = [{ id: 33, url }];
|
||||||
|
const forge = fakeForge([{ id: 4, url, system: false }], repoHooks);
|
||||||
|
const hook = await ensureSystemHook(forge.admin, url, secret);
|
||||||
|
assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] });
|
||||||
|
assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept");
|
||||||
|
assert.deepEqual(forge.repoHooks["novox/later"], []);
|
||||||
|
assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge that makes no system webhook is said, never taken as heard", async () => {
|
||||||
|
const { ensureSystemHook } = await import("../webhook.ts");
|
||||||
|
const forge = fakeForge();
|
||||||
|
const admin = { call: async (method: string, path: string, body?: any) => {
|
||||||
|
// A forge that ignores the flag: what 23:05 met, had the module read back.
|
||||||
|
if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } };
|
||||||
|
return forge.admin.call(method, path, body);
|
||||||
|
} };
|
||||||
|
await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("the receiver tells what it verified and what it refused", async () => {
|
||||||
|
const { receiver, HOOK_PATH } = await import("../webhook.ts");
|
||||||
|
const heard: string[] = [], refused: string[] = [];
|
||||||
|
const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {},
|
||||||
|
heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) });
|
||||||
|
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
|
||||||
|
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`;
|
||||||
|
try {
|
||||||
|
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } });
|
||||||
|
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } });
|
||||||
|
await server.idle();
|
||||||
|
assert.deepEqual(heard, ["pull_request d-9"]);
|
||||||
|
assert.deepEqual(refused, ["its signature does not verify"]);
|
||||||
|
} finally { server.close(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const pace = new Pace(() => now, (l) => said.push(l));
|
||||||
|
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
|
||||||
|
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
|
||||||
|
pace.listening("172.17.0.1", 3047, hook.url, hook);
|
||||||
|
now += 5 * 60_000;
|
||||||
|
pace.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.length, 0, "five minutes after the start is not yet silence");
|
||||||
|
assert.equal((pace.report() as any).healthy, true);
|
||||||
|
pace.refused("it is unsigned", "172.17.0.3");
|
||||||
|
assert.equal((pace.report() as any).healthy, false, "refused, and none verified");
|
||||||
|
now += 6 * 60_000;
|
||||||
|
pace.miss("the merge of novox/hq#7");
|
||||||
|
pace.miss("novox/hq#8 at bbbbbbbb");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once");
|
||||||
|
let r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.equal(r.state, "unhealthy");
|
||||||
|
assert.match(r.why, /never delivered/);
|
||||||
|
assert.equal(r.polls.found_unsaid, 3);
|
||||||
|
assert.equal(r.refused.count, 1);
|
||||||
|
assert.equal(r.refused.last.reason, "it is unsigned");
|
||||||
|
assert.equal(r.hook.kind, "system");
|
||||||
|
assert.equal(r.receiver.port, 3047);
|
||||||
|
// The first delivery clears it.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-1");
|
||||||
|
r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, true);
|
||||||
|
assert.equal(r.verified.count, 1);
|
||||||
|
assert.equal(r.verified.last.event, "push");
|
||||||
|
assert.equal(r.polls.every_seconds, 300);
|
||||||
|
assert.match(said.join("\n"), /heard again/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const pace = new Pace(() => now, (l) => said.push(l));
|
||||||
|
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
|
||||||
|
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
|
||||||
|
pace.listening("172.17.0.1", 3047, hook.url, hook);
|
||||||
|
now += 60_000;
|
||||||
|
pace.heard("push", "d-1");
|
||||||
|
// The hook is deleted by hand: nothing more comes, and the poll finds what no event said.
|
||||||
|
now += 11 * 60_000;
|
||||||
|
pace.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event");
|
||||||
|
pace.miss("the merge of novox/hq#7");
|
||||||
|
pace.miss("novox/hq#8 at bbbbbbbb");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once");
|
||||||
|
assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/);
|
||||||
|
assert.equal((pace.report() as any).healthy, false);
|
||||||
|
// A delivery ends the spell; the next spell is said again.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-2");
|
||||||
|
assert.equal((pace.report() as any).healthy, true);
|
||||||
|
now += 11 * 60_000;
|
||||||
|
pace.miss("novox/hq#9 at cccccccc");
|
||||||
|
pace.miss("novox/hq#10 at dddddddd");
|
||||||
|
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again");
|
||||||
|
// Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once.
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-3");
|
||||||
|
now += 1000;
|
||||||
|
pace.refused("its signature does not verify", "172.17.0.3");
|
||||||
|
pace.refused("its signature does not verify", "172.17.0.3");
|
||||||
|
const r = pace.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.equal(r.refused.since_verified, 2);
|
||||||
|
assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/);
|
||||||
|
now += 1000;
|
||||||
|
pace.heard("push", "d-4");
|
||||||
|
assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => {
|
||||||
|
const { Pace } = await import("../webhook.ts");
|
||||||
|
let now = 1_000_000;
|
||||||
|
const said: string[] = [];
|
||||||
|
const quiet = new Pace(() => now, (l) => said.push(l));
|
||||||
|
quiet.unheard("MESH_GITEA_HOOK_URL unset", false);
|
||||||
|
now += 20 * 60_000;
|
||||||
|
quiet.miss("novox/hq#7 at aaaaaaaa");
|
||||||
|
assert.equal(said.length, 0, "no deliveries were expected");
|
||||||
|
assert.equal((quiet.report() as any).healthy, true);
|
||||||
|
const failed = new Pace(() => now, () => {});
|
||||||
|
failed.unheard("the forge's system webhook could not be made: 403");
|
||||||
|
const r = failed.report() as any;
|
||||||
|
assert.equal(r.healthy, false);
|
||||||
|
assert.match(r.why, /403/);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {
|
test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {
|
||||||
|
|||||||
+208
-78
@@ -12,7 +12,20 @@
|
|||||||
// hands a token's value out exactly once, so a token not kept is a token lost;
|
// hands a token's value out exactly once, so a token not kept is a token lost;
|
||||||
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
|
||||||
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
// a fault: the forge's data was restored from a predecessor and the token the file names never
|
||||||
// existed there.
|
// existed there;
|
||||||
|
// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process
|
||||||
|
// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two
|
||||||
|
// processes over one kept file. Two that minted at once left the forge two tokens by one name, which
|
||||||
|
// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14
|
||||||
|
// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it
|
||||||
|
// holds it: what another process minted meanwhile is reused, not minted over;
|
||||||
|
// - replaced by id: an old token by this name is found in the account's list and deleted by its id,
|
||||||
|
// every one of them, so a name held twice is cleared rather than refused.
|
||||||
|
//
|
||||||
|
// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's
|
||||||
|
// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any
|
||||||
|
// token by this name the administrator itself still holds — the token the tools used before. A token kept
|
||||||
|
// for another account than the one asked (the scopes line says whose) is minted again at the next start.
|
||||||
//
|
//
|
||||||
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is
|
||||||
// reported, not repaired — somebody chose it.
|
// reported, not repaired — somebody chose it.
|
||||||
@@ -20,8 +33,10 @@
|
|||||||
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is
|
||||||
// kept; never what it is.
|
// kept; never what it is.
|
||||||
|
|
||||||
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
||||||
|
import { randomBytes } from "node:crypto";
|
||||||
import { dirname, join } from "node:path";
|
import { dirname, join } from "node:path";
|
||||||
|
import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js";
|
||||||
|
|
||||||
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
|
||||||
export const TOKEN_NAME = "mesh-tools";
|
export const TOKEN_NAME = "mesh-tools";
|
||||||
@@ -39,7 +54,7 @@ export const TOKEN_NAME = "mesh-tools";
|
|||||||
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
|
||||||
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
|
||||||
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
* A token kept with a different list is minted again with this one at the next start (the scopes are kept
|
||||||
* beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this.
|
* beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this.
|
||||||
*
|
*
|
||||||
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
* A token kept from before a scope was added lacks it: the forge answers such a call with
|
||||||
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
* `403 token does not have at least one of required scope(s)`, and the client treats that like a
|
||||||
@@ -95,42 +110,37 @@ export interface MintedTokenOptions {
|
|||||||
readonly file: string;
|
readonly file: string;
|
||||||
readonly name?: string;
|
readonly name?: string;
|
||||||
readonly scopes?: readonly string[];
|
readonly scopes?: readonly string[];
|
||||||
|
/**
|
||||||
|
* The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after
|
||||||
|
* `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the
|
||||||
|
* admin's own, as it was before.
|
||||||
|
*/
|
||||||
|
readonly account?: string;
|
||||||
|
/** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */
|
||||||
|
readonly setUp?: (password: string) => Promise<void>;
|
||||||
readonly log?: (line: string) => void;
|
readonly log?: (line: string) => void;
|
||||||
readonly fetch?: typeof fetch;
|
readonly fetch?: typeof fetch;
|
||||||
/** The clock, in milliseconds; a test sets it. */
|
|
||||||
readonly now?: () => number;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* **How long a failed mint is not tried again** (novox/hq issue 739). Every call that finds no token mints, and a
|
|
||||||
* mint is two or three calls over basic auth, each one hashing the admin password on the forge. A mint that keeps
|
|
||||||
* failing, once per tool call, is a loop the forge pays for in CPU: on 2026-10-14 it ran several times a minute
|
|
||||||
* on the control-node while it was already overloaded. Within this window the last failure is answered again,
|
|
||||||
* without asking the forge.
|
|
||||||
*/
|
|
||||||
export const MINT_RETRY_AFTER_MS = 30_000;
|
|
||||||
|
|
||||||
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
|
||||||
export class MintedToken implements TokenSource {
|
export class MintedToken implements TokenSource {
|
||||||
private held: string | null = null;
|
private held: string | null = null;
|
||||||
private readFile = false;
|
private readFile = false;
|
||||||
private inflight: Promise<string> | null = null;
|
private inflight: Promise<string> | null = null;
|
||||||
private failed: { at: number; error: unknown } | null = null;
|
|
||||||
private readonly name: string;
|
private readonly name: string;
|
||||||
private readonly scopes: readonly string[];
|
private readonly scopes: readonly string[];
|
||||||
|
private readonly account: string;
|
||||||
private readonly log: (line: string) => void;
|
private readonly log: (line: string) => void;
|
||||||
private readonly fetchImpl: typeof fetch;
|
private readonly fetchImpl: typeof fetch;
|
||||||
|
|
||||||
constructor(private readonly opts: MintedTokenOptions) {
|
constructor(private readonly opts: MintedTokenOptions) {
|
||||||
this.name = opts.name ?? TOKEN_NAME;
|
this.name = opts.name ?? TOKEN_NAME;
|
||||||
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
this.scopes = opts.scopes ?? TOKEN_SCOPES;
|
||||||
|
this.account = opts.account ?? opts.admin;
|
||||||
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
|
||||||
this.fetchImpl = opts.fetch ?? fetch;
|
this.fetchImpl = opts.fetch ?? fetch;
|
||||||
this.now = opts.now ?? Date.now;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly now: () => number;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
* Build from the runtime's environment: the forge's URL, the admin login and password file the
|
||||||
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
|
||||||
@@ -139,12 +149,9 @@ export class MintedToken implements TokenSource {
|
|||||||
*/
|
*/
|
||||||
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
|
||||||
const opts = MintedToken.optionsFromEnv(url, env);
|
const opts = MintedToken.optionsFromEnv(url, env);
|
||||||
// One source per kept file in a process. The watcher and the tools entrypoint both build a
|
// One source per kept file in a process: two sources in one process would each renew on a 401.
|
||||||
// client in the same runtime; two sources over one file would each renew on a 401 and drop the
|
// This covers one process only. The watcher and the tools are two processes (the runtime starts
|
||||||
// other's token by name, forever. Shared, a renewal is one renewal. Across processes it is not shared: the
|
// each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below).
|
||||||
// runtime serves the watcher (index.js) and the tools (tools/index.js) as separate bundles, so two sources over
|
|
||||||
// one file do exist. renew() reads the kept file before it mints, and a race that still leaves two tokens by
|
|
||||||
// the name is undone by dropping each by its id (doMint; novox/hq issue 739).
|
|
||||||
const shared = MintedToken.shared.get(opts.file);
|
const shared = MintedToken.shared.get(opts.file);
|
||||||
if (shared) return shared;
|
if (shared) return shared;
|
||||||
const source = new MintedToken(opts);
|
const source = new MintedToken(opts);
|
||||||
@@ -166,7 +173,17 @@ export class MintedToken implements TokenSource {
|
|||||||
if (missing.length) {
|
if (missing.length) {
|
||||||
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
|
||||||
}
|
}
|
||||||
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") };
|
// The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it.
|
||||||
|
const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT;
|
||||||
|
if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`);
|
||||||
|
return {
|
||||||
|
url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account,
|
||||||
|
// GiteaAdmin is loaded when first used: client.ts imports this file.
|
||||||
|
setUp: async (password) => {
|
||||||
|
const { GiteaAdmin } = await import("./client.js");
|
||||||
|
await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url"));
|
||||||
|
},
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async current(): Promise<string> {
|
async current(): Promise<string> {
|
||||||
@@ -177,11 +194,13 @@ export class MintedToken implements TokenSource {
|
|||||||
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
if (kept !== null && this.keptScopes() === this.scopesLine()) {
|
||||||
this.held = kept;
|
this.held = kept;
|
||||||
this.log(`reusing the token kept at ${this.opts.file}`);
|
this.log(`reusing the token kept at ${this.opts.file}`);
|
||||||
|
// A retirement of the admin's old token that failed after the mint is tried again, once a process.
|
||||||
|
if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`));
|
||||||
return kept;
|
return kept;
|
||||||
}
|
}
|
||||||
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them");
|
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept);
|
||||||
}
|
}
|
||||||
return this.mint(this.failed ? "no token held since the last mint failed — minting one" : "no token kept — minting one");
|
return this.mint("no token kept — minting one", null);
|
||||||
}
|
}
|
||||||
|
|
||||||
async renew(rejected: string): Promise<string> {
|
async renew(rejected: string): Promise<string> {
|
||||||
@@ -195,7 +214,7 @@ export class MintedToken implements TokenSource {
|
|||||||
return kept;
|
return kept;
|
||||||
}
|
}
|
||||||
this.held = null;
|
this.held = null;
|
||||||
return this.mint("the forge rejected the kept token — minting a fresh one");
|
return this.mint("the forge rejected the kept token — minting a fresh one", rejected);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
|
||||||
@@ -203,28 +222,95 @@ export class MintedToken implements TokenSource {
|
|||||||
return status === 403 && /required scope/i.test(body);
|
return status === 403 && /required scope/i.test(body);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */
|
/**
|
||||||
private mint(why: string): Promise<string> {
|
* One mint at a time: concurrent first calls in this process share it, and a mint in another process
|
||||||
|
* on this machine is waited for (the lock file). Once the lock is held, the kept file is read again:
|
||||||
|
* a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the
|
||||||
|
* scopes asked for was minted by the process that held the lock before — it is reused.
|
||||||
|
*/
|
||||||
|
private mint(why: string, stale: string | null): Promise<string> {
|
||||||
if (this.inflight === null) {
|
if (this.inflight === null) {
|
||||||
// A mint that failed a moment ago is not asked of the forge again yet (novox/hq issue 739): its error stands.
|
|
||||||
if (this.failed !== null && this.now() - this.failed.at < MINT_RETRY_AFTER_MS) return Promise.reject(this.failed.error);
|
|
||||||
this.log(why);
|
this.log(why);
|
||||||
this.inflight = this.doMint().then(
|
this.inflight = this.locked(async () => {
|
||||||
(token) => {
|
const kept = this.read();
|
||||||
this.failed = null;
|
if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) {
|
||||||
return token;
|
this.held = kept;
|
||||||
},
|
this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`);
|
||||||
(err) => {
|
return kept;
|
||||||
this.failed = { at: this.now(), error: err };
|
}
|
||||||
throw err;
|
return this.doMint();
|
||||||
},
|
}).finally(() => {
|
||||||
).finally(() => {
|
|
||||||
this.inflight = null;
|
this.inflight = null;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return this.inflight;
|
return this.inflight;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */
|
||||||
|
static readonly LOCK_STALE_MS = 120_000;
|
||||||
|
static readonly LOCK_WAIT_MS = 150_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run `work` holding the lock file beside the kept token: created exclusively, holding this process's
|
||||||
|
* pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone,
|
||||||
|
* or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never
|
||||||
|
* waited on for ever, which would be the same lock-out by another road.
|
||||||
|
*/
|
||||||
|
private async locked<T>(work: () => Promise<T>): Promise<T> {
|
||||||
|
const lock = `${this.opts.file}.lock`;
|
||||||
|
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
|
||||||
|
const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`;
|
||||||
|
const deadline = Date.now() + MintedToken.LOCK_WAIT_MS;
|
||||||
|
for (;;) {
|
||||||
|
try {
|
||||||
|
const fd = openSync(lock, "wx", 0o600);
|
||||||
|
try { writeSync(fd, mine); } finally { closeSync(fd); }
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
|
||||||
|
throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (this.leftBehind(lock)) {
|
||||||
|
this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`);
|
||||||
|
try { unlinkSync(lock); } catch { /* another waiter took it over first */ }
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (Date.now() > deadline) {
|
||||||
|
throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`);
|
||||||
|
}
|
||||||
|
await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50)));
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await work();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
if (readFileSync(lock, "utf8") === mine) unlinkSync(lock);
|
||||||
|
} catch { /* already gone */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */
|
||||||
|
private leftBehind(lock: string): boolean {
|
||||||
|
let text: string;
|
||||||
|
let age: number;
|
||||||
|
try {
|
||||||
|
text = readFileSync(lock, "utf8");
|
||||||
|
age = Date.now() - statSync(lock).mtimeMs;
|
||||||
|
} catch {
|
||||||
|
return false; // released meanwhile: the next attempt takes it
|
||||||
|
}
|
||||||
|
if (age > MintedToken.LOCK_STALE_MS) return true;
|
||||||
|
const pid = Number(text.split(" ")[0]);
|
||||||
|
if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment
|
||||||
|
try {
|
||||||
|
process.kill(pid, 0);
|
||||||
|
return false;
|
||||||
|
} catch (err) {
|
||||||
|
return (err as NodeJS.ErrnoException).code === "ESRCH";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private read(): string | null {
|
private read(): string | null {
|
||||||
try {
|
try {
|
||||||
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
|
||||||
@@ -235,9 +321,10 @@ export class MintedToken implements TokenSource {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The scopes as kept beside the token: sorted, one line. */
|
/** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */
|
||||||
private scopesLine(): string {
|
private scopesLine(): string {
|
||||||
return [...this.scopes].sort().join(",");
|
const scopes = [...this.scopes].sort().join(",");
|
||||||
|
return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
|
||||||
@@ -260,17 +347,20 @@ export class MintedToken implements TokenSource {
|
|||||||
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async doMint(): Promise<string> {
|
/** One call to the forge's API with the admin login, the password read now. */
|
||||||
let password: string;
|
private password(): string {
|
||||||
try {
|
try {
|
||||||
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private adminCall(password = this.password()): AdminCall {
|
||||||
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
|
||||||
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`;
|
const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`;
|
||||||
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => {
|
const admin: AdminCall = async (method, path, body) => {
|
||||||
const res = await this.fetchImpl(tokens + path, {
|
const res = await this.fetchImpl(api + path, {
|
||||||
method,
|
method,
|
||||||
headers: { "Content-Type": "application/json", Authorization: authorization },
|
headers: { "Content-Type": "application/json", Authorization: authorization },
|
||||||
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
||||||
@@ -282,56 +372,96 @@ export class MintedToken implements TokenSource {
|
|||||||
}
|
}
|
||||||
return { status: res.status, body: parsed };
|
return { status: res.status, body: parsed };
|
||||||
};
|
};
|
||||||
|
return admin;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */
|
||||||
|
private get retireFile(): string {
|
||||||
|
return `${this.opts.file}.retire-admin`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR
|
||||||
|
* 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without
|
||||||
|
* a token: it is said, remembered beside the kept token, and tried again when a process next reads that token.
|
||||||
|
*/
|
||||||
|
private async retireAdmin(admin: AdminCall = this.adminCall()): Promise<void> {
|
||||||
|
if (this.account === this.opts.admin) return;
|
||||||
|
try {
|
||||||
|
const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin);
|
||||||
|
if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`);
|
||||||
|
rmSync(this.retireFile, { force: true });
|
||||||
|
} catch (err) {
|
||||||
|
writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 });
|
||||||
|
this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async doMint(): Promise<string> {
|
||||||
|
const password = this.password();
|
||||||
|
const admin = this.adminCall(password);
|
||||||
|
const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`;
|
||||||
|
const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body);
|
||||||
|
|
||||||
|
if (this.account !== this.opts.admin) {
|
||||||
|
// The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again
|
||||||
|
// at the next call.
|
||||||
|
const me = await admin("GET", "/user");
|
||||||
|
if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status);
|
||||||
|
await this.opts.setUp?.(password);
|
||||||
|
}
|
||||||
|
|
||||||
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
let res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
||||||
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
|
||||||
if (res.status === 400 || res.status === 422) {
|
if (res.status === 400 || res.status === 422) {
|
||||||
// The forge still holds a token by this name whose value we no longer have — the kept file
|
// The forge still holds a token by this name whose value we no longer have — the kept file
|
||||||
// went while the forge's data stayed. It is ours to replace: drop it by name and mint again.
|
// went while the forge's data stayed, or two processes once minted at once and left two. It is
|
||||||
this.log(`the forge already holds a token named "${this.name}" — replacing it`);
|
// ours to replace: find every one by this name in the account's list and delete each by its id
|
||||||
const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`);
|
// (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint.
|
||||||
if (dropped.status === 422) {
|
await this.dropNamed(admin, tokensOf(this.account), this.account, true);
|
||||||
// More than one token carries the name (novox/hq issue 739): two processes minted at once, and the forge's
|
|
||||||
// check that a name is unused is not atomic. The forge refuses a delete by such a name for good, so every
|
|
||||||
// one of them is dropped by its id — all are ours, and none of their values is kept.
|
|
||||||
await this.dropEveryNamed(call, dropped);
|
|
||||||
} else if (dropped.status !== 204 && dropped.status !== 404) {
|
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
|
|
||||||
}
|
|
||||||
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
res = await call("POST", "", { name: this.name, scopes: this.scopes });
|
||||||
}
|
}
|
||||||
if (res.status !== 201 && res.status !== 200) {
|
if (res.status !== 201 && res.status !== 200) {
|
||||||
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`);
|
throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`);
|
||||||
}
|
}
|
||||||
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
|
||||||
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`);
|
if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`);
|
||||||
|
|
||||||
this.keep(token);
|
this.keep(token);
|
||||||
this.held = token;
|
this.held = token;
|
||||||
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
|
||||||
|
// Only now, with the account's own token kept, does the admin's old one go.
|
||||||
|
await this.retireAdmin(admin);
|
||||||
return token;
|
return token;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Drop, by id, every token of the admin account that carries this name; the forge listed them, page by page. */
|
/**
|
||||||
private async dropEveryNamed(call: (method: string, path?: string) => Promise<{ status: number; body: any }>,
|
* Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice:
|
||||||
refused: { status: number; body: any }): Promise<void> {
|
* "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name
|
||||||
const ids: number[] = [];
|
* taken.
|
||||||
for (let page = 1; page <= 20; page++) {
|
*/
|
||||||
const listed = await call("GET", `?page=${page}&limit=50`);
|
private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise<number> {
|
||||||
|
const held: number[] = [];
|
||||||
|
for (let page = 1; page <= 100; page++) {
|
||||||
|
const listed = await admin("GET", `${tokens}?page=${page}&limit=50`);
|
||||||
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
if (listed.status !== 200 || !Array.isArray(listed.body)) {
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${refused.status} ${detail(refused.body)}; ` +
|
throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`);
|
||||||
`and its tokens could not be listed to drop them one by one: ${listed.status} ${detail(listed.body)}`);
|
|
||||||
}
|
}
|
||||||
for (const t of listed.body) if (t?.name === this.name && Number.isInteger(t?.id)) ids.push(t.id);
|
if (listed.body.length === 0) break;
|
||||||
if (listed.body.length < 50) break;
|
for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id);
|
||||||
}
|
}
|
||||||
this.log(`the forge holds ${ids.length} tokens named "${this.name}" — dropping each by its id`);
|
if (replacing) {
|
||||||
for (const id of ids) {
|
this.log(held.length === 1
|
||||||
const dropped = await call("DELETE", `/${id}`);
|
? `the forge already holds a token named "${this.name}" — replacing it`
|
||||||
|
: `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`);
|
||||||
|
}
|
||||||
|
for (const id of held) {
|
||||||
|
const dropped = await admin("DELETE", `${tokens}/${id}`);
|
||||||
if (dropped.status !== 204 && dropped.status !== 404) {
|
if (dropped.status !== 204 && dropped.status !== 404) {
|
||||||
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id}: ${dropped.status} ${detail(dropped.body)}`);
|
throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return held.length;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// The forge's tools act as an account of their own, not as the site administrator (novox/hq ADR 0365 §2,
|
||||||
|
// issue 469).
|
||||||
|
//
|
||||||
|
// The administrator's token reached everything the forge holds, so the tools' boundary rested on their own guards
|
||||||
|
// (guard.ts): a protection edit, say, the forge itself would have taken. Now the tools' token is minted for a plain
|
||||||
|
// account, `mesh-tools`, which is no administrator and writes to the mesh's repositories through one team of the
|
||||||
|
// mesh's organisation:
|
||||||
|
//
|
||||||
|
// - **the account**: made by the module with the administrator's login (that login sets the forge up and does
|
||||||
|
// nothing else), never an administrator — one found made an administrator is made a plain account again. Its
|
||||||
|
// password is random, sent once and kept nowhere: the account is reached only by the tokens the administrator
|
||||||
|
// mints for it (gitea lets an administrator mint for another account: `reqSelfOrAdmin`).
|
||||||
|
// - **the team**: `mesh-tools` in the organisation `novox`, on every repository of it, writing code (pushes to a
|
||||||
|
// branch, commit statuses, merges), issues (notes, comments, labels) and pull requests. No administration of a
|
||||||
|
// repository, so the forge refuses the account a protection edit, a repository's settings and its deletion;
|
||||||
|
// no repository made in the organisation, since its maker would be that repository's administrator.
|
||||||
|
//
|
||||||
|
// Every step is idempotent (GiteaAdmin's, client.ts), and run with the administrator's login before each mint of the tools' token, so a team
|
||||||
|
// or a membership lost behind the module's back is made again at the next mint.
|
||||||
|
|
||||||
|
/** The account the tools act as. */
|
||||||
|
export const TOOLS_ACCOUNT = "mesh-tools";
|
||||||
|
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
|
||||||
|
export const TOOLS_ORG = "novox";
|
||||||
|
export const TOOLS_TEAM = "mesh-tools";
|
||||||
|
|
||||||
|
/** What the team grants: write on code, issues and pull requests of every repository of the organisation, and no more. */
|
||||||
|
export const TOOLS_TEAM_UNITS = {
|
||||||
|
permission: "write",
|
||||||
|
units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" },
|
||||||
|
includes_all_repositories: true,
|
||||||
|
can_create_org_repo: false,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
/** One call to the forge's API with the administrator's login: a path under /api/v1, the status and body answered. */
|
||||||
|
export type AdminCall = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: any }>;
|
||||||
|
|
||||||
|
/** What setting the account up needs of the forge's admin API: GiteaAdmin's own (client.ts), shared with the
|
||||||
|
* package provisioner, so a create race (422) and a user made by hand are handled once. */
|
||||||
|
export interface ToolsForge {
|
||||||
|
ensureUser(username: string, password: string, email: string, plain?: boolean): Promise<void>;
|
||||||
|
ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number>;
|
||||||
|
addUserToTeam(teamId: number, username: string): Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Make the tools' account, plain, a member of its team on every repository of the organisation. Its password is
|
||||||
|
* set afresh each time and kept nowhere: the account is reached only by the tokens the administrator mints. */
|
||||||
|
export async function setUpToolsAccount(forge: ToolsForge, account = TOOLS_ACCOUNT, password: () => string): Promise<void> {
|
||||||
|
await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true);
|
||||||
|
const team = await forge.ensureTeamWith(TOOLS_ORG, TOOLS_TEAM, { name: TOOLS_TEAM, ...TOOLS_TEAM_UNITS });
|
||||||
|
await forge.addUserToTeam(team, account);
|
||||||
|
}
|
||||||
@@ -10,7 +10,7 @@
|
|||||||
|
|
||||||
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
|
||||||
import { emit } from "@novox/mesh-sdk/events";
|
import { emit } from "@novox/mesh-sdk/events";
|
||||||
import { GiteaClient, readyAuthors, type BranchProtection } from "../client.js";
|
import { GiteaAdmin, GiteaClient, readyAuthors, type BranchProtection } from "../client.js";
|
||||||
import { appendNote, run, viewBody, viewComment } from "../delivery.js";
|
import { appendNote, run, viewBody, viewComment } from "../delivery.js";
|
||||||
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
|
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
|
||||||
import {
|
import {
|
||||||
@@ -59,13 +59,13 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] {
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||||
return namesChecked(allGiteaTools(gitea));
|
return namesChecked(allGiteaTools(gitea, admin));
|
||||||
}
|
}
|
||||||
|
|
||||||
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
|
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
|
||||||
// writer (novox/hq issue 456, ADR 0323).
|
// writer (novox/hq issue 456, ADR 0323).
|
||||||
function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
|
||||||
return [
|
return [
|
||||||
// ---- Repositories ----
|
// ---- Repositories ----
|
||||||
{
|
{
|
||||||
@@ -401,8 +401,11 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
},
|
},
|
||||||
run: async (args) => {
|
run: async (args) => {
|
||||||
const owner = String(args.owner), repo = String(args.repo);
|
const owner = String(args.owner), repo = String(args.repo);
|
||||||
if (!args.branch) return { rules: (await gitea.branchProtections(owner, repo)).map(summarised) };
|
// The one read made as the forge's administrator (novox/hq ADR 0375, issue 772): the rules' list is the
|
||||||
const rule = await gitea.branchProtection(owner, repo, String(args.branch));
|
// repository administrator's, which the tools' account is not. No other tool uses this account.
|
||||||
|
const rules = await (admin ? admin() : gitea).branchProtections(owner, repo);
|
||||||
|
if (!args.branch) return { rules: rules.map(summarised) };
|
||||||
|
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === String(args.branch)) ?? null;
|
||||||
return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
|
return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -504,7 +507,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
base = base || "main";
|
base = base || "main";
|
||||||
let rule: BranchProtection | null = null, unreadable: string | undefined;
|
let rule: BranchProtection | null = null, unreadable: string | undefined;
|
||||||
try {
|
try {
|
||||||
rule = await gitea.branchProtection(owner, repo, base);
|
rule = await gitea.appliedProtection(owner, repo, base);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
unreadable = err instanceof Error ? err.message : String(err);
|
unreadable = err instanceof Error ? err.message : String(err);
|
||||||
}
|
}
|
||||||
@@ -589,8 +592,12 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
const { owner, repo, branch, newBranch } = judged.contents;
|
const { owner, repo, branch, newBranch } = judged.contents;
|
||||||
const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`);
|
const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`);
|
||||||
const defaultBranch = String(found?.default_branch ?? "main");
|
const defaultBranch = String(found?.default_branch ?? "main");
|
||||||
const refused = judgeContentsWrite(newBranch ?? branch ?? defaultBranch, defaultBranch,
|
// The protection the forge applies to the branch written, read from the branch: the rules' own list is a
|
||||||
await gitea.branchProtections(owner, repo));
|
// repository administrator's, which the tools' account is not (novox/hq ADR 0365). A branch not made yet is
|
||||||
|
// left to the forge, which refuses a write that would make a branch a rule guards.
|
||||||
|
const target = newBranch ?? branch ?? defaultBranch;
|
||||||
|
const applied = target === defaultBranch ? null : await gitea.appliedProtection(owner, repo, target);
|
||||||
|
const refused = judgeContentsWrite(target, defaultBranch, applied ? [applied] : []);
|
||||||
if (refused) throw new Refused(refused);
|
if (refused) throw new Refused(refused);
|
||||||
}
|
}
|
||||||
// The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts).
|
// The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts).
|
||||||
@@ -611,7 +618,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
|
|||||||
// way, gitea contributes none rather than failing the whole runtime, and says why.
|
// way, gitea contributes none rather than failing the whole runtime, and says why.
|
||||||
registerModuleTools("gitea", (env) => {
|
registerModuleTools("gitea", (env) => {
|
||||||
try {
|
try {
|
||||||
return getGiteaTools(GiteaClient.fromEnv(env));
|
return getGiteaTools(GiteaClient.fromEnv(env), () => GiteaAdmin.fromEnv(env));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`);
|
console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`);
|
||||||
return [];
|
return [];
|
||||||
|
|||||||
@@ -128,10 +128,10 @@ export interface OutOfDateWriter {
|
|||||||
* out of date (ADR 0323 §4). A trunk first seen is recorded as it is and nothing is put out of date. Bounded however
|
* out of date (ADR 0323 §4). A trunk first seen is recorded as it is and nothing is put out of date. Bounded however
|
||||||
* many are open: one listing of the open pull requests (paged), one read of each trunk, then per open pull request into
|
* many are open: one listing of the open pull requests (paged), one read of each trunk, then per open pull request into
|
||||||
* a trunk that moved one read and at most two writes, one at a time. A trunk that could not be weighed whole is not
|
* a trunk that moved one read and at most two writes, one at a time. A trunk that could not be weighed whole is not
|
||||||
* recorded, so the next pass weighs it again. Answers whether the record changed.
|
* recorded, so the next pass weighs it again. Answers whether the record changed. `moved` is told of each base that moved, for the siblings whose base it is.
|
||||||
*/
|
*/
|
||||||
export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, trunks: Record<string, string>, owner: string,
|
export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, trunks: Record<string, string>, owner: string,
|
||||||
name: string): Promise<boolean> {
|
name: string, moved?: (base: string) => Promise<void>): Promise<boolean> {
|
||||||
let changed = false;
|
let changed = false;
|
||||||
try {
|
try {
|
||||||
const open = (await forge.listOpenPulls(owner, name)).filter((p) => p.state === "open" && !!p.head_sha && !!p.base);
|
const open = (await forge.listOpenPulls(owner, name)).filter((p) => p.state === "open" && !!p.head_sha && !!p.base);
|
||||||
@@ -151,6 +151,8 @@ export async function weighTrunks(forge: TrunkForge, gates: OutOfDateWriter, tru
|
|||||||
}
|
}
|
||||||
console.log(`[gitea] ${owner}/${name}'s ${base} moved from ${twelve(before)} to ${twelve(head)}: ${put} open pull ` +
|
console.log(`[gitea] ${owner}/${name}'s ${base} moved from ${twelve(before)} to ${twelve(head)}: ${put} open pull ` +
|
||||||
`request(s) into it put out of date, nothing run (novox/hq ADR 0323)`);
|
`request(s) into it put out of date, nothing run (novox/hq ADR 0323)`);
|
||||||
|
// A verdict that read a pull request into this trunk as a sibling is out of date by its move too (ADR 0382 §5).
|
||||||
|
if (moved) await moved(base);
|
||||||
}
|
}
|
||||||
trunks[key] = head;
|
trunks[key] = head;
|
||||||
changed = true;
|
changed = true;
|
||||||
|
|||||||
+241
-36
@@ -140,8 +140,11 @@ export interface ReceiverOptions {
|
|||||||
publish: (event: ForgeEvent) => Promise<void>;
|
publish: (event: ForgeEvent) => Promise<void>;
|
||||||
/** Acts on it at once, beside publishing it (the announcer). */
|
/** Acts on it at once, beside publishing it (the announcer). */
|
||||||
act?: (event: ForgeEvent) => Promise<void>;
|
act?: (event: ForgeEvent) => Promise<void>;
|
||||||
/** Told of every verified delivery, so the polls know events are heard. */
|
/** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are
|
||||||
heard?: () => void;
|
* heard, and the module can say when it last heard one. */
|
||||||
|
heard?: (kind: string, delivery: string) => void;
|
||||||
|
/** Told of every refused delivery, with the reason and where it came from. */
|
||||||
|
refused?: (reason: string, from: string) => void;
|
||||||
log?: (line: string) => void;
|
log?: (line: string) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -158,6 +161,10 @@ export const REMEMBERED = 2000;
|
|||||||
*/
|
*/
|
||||||
export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
|
export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
|
||||||
const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
|
const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
|
||||||
|
const refuse = (from: string, reason: string): void => {
|
||||||
|
log(`refused a forge event from ${from}: ${reason}`);
|
||||||
|
opts.refused?.(reason, from);
|
||||||
|
};
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
let queue: Promise<void> = Promise.resolve();
|
let queue: Promise<void> = Promise.resolve();
|
||||||
const server = createServer((req, res) => {
|
const server = createServer((req, res) => {
|
||||||
@@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
const from = req.socket.remoteAddress ?? "?";
|
const from = req.socket.remoteAddress ?? "?";
|
||||||
const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
|
const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
|
||||||
if (!signature) {
|
if (!signature) {
|
||||||
log(`refused a forge event from ${from}: it is unsigned`);
|
refuse(from, "it is unsigned");
|
||||||
res.writeHead(401, { Connection: "close" });
|
res.writeHead(401, { Connection: "close" });
|
||||||
res.end("unsigned\n");
|
res.end("unsigned\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
|
if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
|
||||||
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
|
refuse(from, `its body is over ${MAX_BODY} bytes`);
|
||||||
res.writeHead(413, { Connection: "close" });
|
res.writeHead(413, { Connection: "close" });
|
||||||
res.end("too large\n");
|
res.end("too large\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
@@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
if (size > MAX_BODY) {
|
if (size > MAX_BODY) {
|
||||||
if (!over) {
|
if (!over) {
|
||||||
over = true;
|
over = true;
|
||||||
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`);
|
refuse(from, `its body is over ${MAX_BODY} bytes`);
|
||||||
res.writeHead(413, { Connection: "close" });
|
res.writeHead(413, { Connection: "close" });
|
||||||
res.end("too large\n");
|
res.end("too large\n");
|
||||||
req.destroy();
|
req.destroy();
|
||||||
@@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
if (over) return;
|
if (over) return;
|
||||||
const body = Buffer.concat(chunks);
|
const body = Buffer.concat(chunks);
|
||||||
if (!verified(opts.secret, body, signature)) {
|
if (!verified(opts.secret, body, signature)) {
|
||||||
log(`refused a forge event from ${from}: its signature does not verify`);
|
refuse(from, "its signature does not verify");
|
||||||
res.statusCode = 401;
|
res.statusCode = 401;
|
||||||
return res.end("signature does not verify\n");
|
return res.end("signature does not verify\n");
|
||||||
}
|
}
|
||||||
@@ -208,11 +215,13 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
try {
|
try {
|
||||||
payload = JSON.parse(body.toString("utf8"));
|
payload = JSON.parse(body.toString("utf8"));
|
||||||
} catch {
|
} catch {
|
||||||
|
refuse(from, "it is signed but not JSON");
|
||||||
res.statusCode = 400;
|
res.statusCode = 400;
|
||||||
return res.end("not JSON\n");
|
return res.end("not JSON\n");
|
||||||
}
|
}
|
||||||
opts.heard?.();
|
|
||||||
const delivery = String(req.headers["x-gitea-delivery"] ?? "");
|
const delivery = String(req.headers["x-gitea-delivery"] ?? "");
|
||||||
|
const kind = String(req.headers["x-gitea-event"] ?? "other");
|
||||||
|
opts.heard?.(kind, delivery);
|
||||||
if (delivery && seen.has(delivery)) {
|
if (delivery && seen.has(delivery)) {
|
||||||
res.statusCode = 200;
|
res.statusCode = 200;
|
||||||
return res.end("already heard\n");
|
return res.end("already heard\n");
|
||||||
@@ -221,7 +230,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
|
|||||||
seen.add(delivery);
|
seen.add(delivery);
|
||||||
if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
|
if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
|
||||||
}
|
}
|
||||||
const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload);
|
const event = eventOf(kind, payload);
|
||||||
res.statusCode = 200;
|
res.statusCode = 200;
|
||||||
res.end("queued\n");
|
res.end("queued\n");
|
||||||
queue = queue.then(async () => {
|
queue = queue.then(async () => {
|
||||||
@@ -242,27 +251,113 @@ export interface HookAdmin {
|
|||||||
call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
|
call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */
|
||||||
|
export interface HookState {
|
||||||
|
/** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */
|
||||||
|
kind: "system";
|
||||||
|
id: number;
|
||||||
|
url: string;
|
||||||
|
events: string[];
|
||||||
|
active: boolean;
|
||||||
|
content_type?: string;
|
||||||
|
/** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */
|
||||||
|
did: "created" | "replaced";
|
||||||
|
/** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which
|
||||||
|
* would have sent that repository's events a second time. */
|
||||||
|
removed: { system: number; default: number; repository: number; repositories: string[] };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll
|
||||||
|
* finds what no event said. */
|
||||||
|
export const SILENT_AFTER = 10 * 60_000;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified
|
* **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the
|
||||||
* delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one,
|
* forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first
|
||||||
* or once the poll found something no event said, every thirty seconds.
|
* delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds.
|
||||||
|
*
|
||||||
|
* It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused
|
||||||
|
* deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver
|
||||||
|
* listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery —
|
||||||
|
* and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an
|
||||||
|
* event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a
|
||||||
|
* delivery verifies. Deliveries refused since the last verified one are said unhealthy too.
|
||||||
*/
|
*/
|
||||||
export class Pace {
|
export class Pace {
|
||||||
private last = 0;
|
private last = 0;
|
||||||
private missed = 0;
|
private missed = 0;
|
||||||
|
private readonly started: number;
|
||||||
private readonly now: () => number;
|
private readonly now: () => number;
|
||||||
private readonly log: (l: string) => void;
|
private readonly log: (l: string) => void;
|
||||||
|
private where: { host: string; port: number; url: string } | null = null;
|
||||||
|
private hook: HookState | null = null;
|
||||||
|
private notHeard: string | null = null;
|
||||||
|
private notHeardFails = false;
|
||||||
|
private verifiedCount = 0;
|
||||||
|
private lastVerified: { at: number; event: string; delivery: string } | null = null;
|
||||||
|
private refusedCount = 0;
|
||||||
|
private refusedSpell = 0;
|
||||||
|
private lastRefused: { at: number; reason: string; from: string } | null = null;
|
||||||
|
private missCount = 0;
|
||||||
|
private lastMiss: { at: number; what: string } | null = null;
|
||||||
|
private silentSince = 0;
|
||||||
|
private quietMisses = 0;
|
||||||
constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
|
constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
|
||||||
this.now = now;
|
this.now = now;
|
||||||
this.log = log;
|
this.log = log;
|
||||||
|
this.started = now();
|
||||||
}
|
}
|
||||||
heard(): void {
|
/** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */
|
||||||
|
listening(host: string, port: number, url: string, hook: HookState): void {
|
||||||
|
this.where = { host, port, url };
|
||||||
|
this.hook = hook;
|
||||||
|
this.notHeard = null;
|
||||||
|
}
|
||||||
|
/** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen
|
||||||
|
* or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */
|
||||||
|
unheard(why: string, fails = true): void {
|
||||||
|
this.notHeard = why;
|
||||||
|
this.notHeardFails = fails;
|
||||||
|
}
|
||||||
|
heard(event = "", delivery = ""): void {
|
||||||
this.last = this.now();
|
this.last = this.now();
|
||||||
|
this.verifiedCount++;
|
||||||
|
this.lastVerified = { at: this.last, event, delivery };
|
||||||
|
this.refusedSpell = 0;
|
||||||
|
this.quietMisses = 0;
|
||||||
|
if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`);
|
||||||
|
this.silentSince = 0;
|
||||||
|
}
|
||||||
|
refused(reason: string, from: string): void {
|
||||||
|
this.refusedCount++;
|
||||||
|
this.refusedSpell++;
|
||||||
|
this.lastRefused = { at: this.now(), reason, from };
|
||||||
}
|
}
|
||||||
/** The poll found a head or merge no forge event announced. */
|
/** The poll found a head or merge no forge event announced. */
|
||||||
miss(what: string): void {
|
miss(what: string): void {
|
||||||
|
const t = this.now();
|
||||||
if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
|
if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
|
||||||
this.missed = this.now();
|
this.missed = t;
|
||||||
|
this.missCount++;
|
||||||
|
this.lastMiss = { at: t, what };
|
||||||
|
// Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by
|
||||||
|
// hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell.
|
||||||
|
const quietSince = this.last || this.started;
|
||||||
|
this.quietMisses++;
|
||||||
|
if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) {
|
||||||
|
this.silentSince = t;
|
||||||
|
const minutes = Math.round((t - quietSince) / 60_000);
|
||||||
|
const refusedSince = this.refusedSinceHeard();
|
||||||
|
this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` +
|
||||||
|
`${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` +
|
||||||
|
`system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` +
|
||||||
|
`${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` +
|
||||||
|
`(novox/hq issue 729)`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/** Whether a delivery was refused after the last verified one (or with none verified). */
|
||||||
|
private refusedSinceHeard(): number {
|
||||||
|
return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0;
|
||||||
}
|
}
|
||||||
hearing(): boolean {
|
hearing(): boolean {
|
||||||
const t = this.now();
|
const t = this.now();
|
||||||
@@ -271,36 +366,146 @@ export class Pace {
|
|||||||
every(): number {
|
every(): number {
|
||||||
return this.hearing() ? 5 * 60_000 : 30_000;
|
return this.hearing() ? 5 * 60_000 : 30_000;
|
||||||
}
|
}
|
||||||
|
private expecting(): boolean {
|
||||||
|
return this.where !== null && this.hook !== null;
|
||||||
|
}
|
||||||
|
/** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */
|
||||||
|
report(): Record<string, unknown> {
|
||||||
|
const t = this.now();
|
||||||
|
const iso = (ms: number): string => new Date(ms).toISOString();
|
||||||
|
let state: "healthy" | "unhealthy" | "waits" = "healthy";
|
||||||
|
let why: string;
|
||||||
|
const refusedSince = this.refusedSinceHeard();
|
||||||
|
if (this.silentSince) {
|
||||||
|
state = "unhealthy";
|
||||||
|
why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` +
|
||||||
|
`event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` +
|
||||||
|
`event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`;
|
||||||
|
} else if (refusedSince) {
|
||||||
|
state = "unhealthy";
|
||||||
|
why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` :
|
||||||
|
"and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`;
|
||||||
|
} else if (!this.expecting()) {
|
||||||
|
why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` :
|
||||||
|
"the receiver is not listening yet";
|
||||||
|
if (this.notHeard && this.notHeardFails) state = "unhealthy";
|
||||||
|
} else if (this.hearing()) {
|
||||||
|
why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`;
|
||||||
|
} else if (this.last === 0) {
|
||||||
|
why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`;
|
||||||
|
} else {
|
||||||
|
why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
healthy: state !== "unhealthy", state, why,
|
||||||
|
started: iso(this.started),
|
||||||
|
receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" },
|
||||||
|
hook: this.hook,
|
||||||
|
verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } },
|
||||||
|
refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } },
|
||||||
|
polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount,
|
||||||
|
last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } },
|
||||||
|
silent: this.silentSince ? { since: iso(this.silentSince) } : null,
|
||||||
|
};
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Every event family the forge (1.27) can send a webhook. */
|
/** Every event family the forge (1.27) can send a webhook. */
|
||||||
export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
|
export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
|
||||||
"pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
|
"pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
|
||||||
|
|
||||||
/**
|
async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise<any[]> {
|
||||||
* **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected
|
const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`);
|
||||||
* when one does but differs (events, active, content type), and its secret set again each start — the forge never
|
if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`);
|
||||||
* hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did.
|
return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url);
|
||||||
*/
|
}
|
||||||
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> {
|
|
||||||
// POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The
|
/** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */
|
||||||
// forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
|
const PAGE = 50;
|
||||||
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
|
async function everyPage(admin: HookAdmin, path: string, what: string): Promise<any[]> {
|
||||||
config: { url, content_type: "json", secret } };
|
const all: any[] = [];
|
||||||
const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50");
|
for (let page = 1; ; page++) {
|
||||||
if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`);
|
const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`);
|
||||||
const hooks: any[] = Array.isArray(listed.body) ? listed.body : [];
|
if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`);
|
||||||
const ours = hooks.filter((h) => h?.config?.url === url);
|
const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : [];
|
||||||
if (ours.length === 0) {
|
all.push(...items);
|
||||||
const made = await admin.call("POST", "/admin/hooks", want);
|
if (items.length < PAGE) return all;
|
||||||
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
|
|
||||||
return "created";
|
|
||||||
}
|
}
|
||||||
for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`);
|
}
|
||||||
const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*",
|
|
||||||
config: { url, content_type: "json", secret } });
|
/** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */
|
||||||
if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`);
|
async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> {
|
||||||
return "corrected";
|
const found: { repo: string; id: number }[] = [];
|
||||||
|
for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) {
|
||||||
|
const full = String(repo?.full_name ?? "");
|
||||||
|
if (!full.includes("/")) continue;
|
||||||
|
const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`);
|
||||||
|
for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) });
|
||||||
|
}
|
||||||
|
return found;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* **One system webhook, for every event, at the receiver** (issue 729), made each start and read back.
|
||||||
|
*
|
||||||
|
* The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless
|
||||||
|
* `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories
|
||||||
|
* made after it, and the listing of system webhooks does not name it — so without the flag every start made another
|
||||||
|
* one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is
|
||||||
|
* not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a
|
||||||
|
* gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a
|
||||||
|
* default one, which would send that repository's events twice. The forge's listing of system webhooks must then name
|
||||||
|
* exactly one at the url, or this fails, saying so.
|
||||||
|
*/
|
||||||
|
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<HookState> {
|
||||||
|
// The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
|
||||||
|
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
|
||||||
|
config: { url, content_type: "json", secret, is_system_webhook: "true" } };
|
||||||
|
const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id)));
|
||||||
|
const made = await admin.call("POST", "/admin/hooks", want);
|
||||||
|
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
|
||||||
|
const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id)));
|
||||||
|
if (fresh.length !== 1) {
|
||||||
|
throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` +
|
||||||
|
`are not known to be sent`);
|
||||||
|
}
|
||||||
|
const ours = fresh[0];
|
||||||
|
const defaults = await hooksAt(admin, "default", url);
|
||||||
|
const gone = async (path: string): Promise<void> => {
|
||||||
|
const r = await admin.call("DELETE", path);
|
||||||
|
if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`);
|
||||||
|
};
|
||||||
|
for (const id of before) await gone(`/admin/hooks/${id}`);
|
||||||
|
for (const h of defaults) await gone(`/admin/hooks/${h.id}`);
|
||||||
|
const copies = await repoHooksAt(admin, url);
|
||||||
|
for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`);
|
||||||
|
const kept = await hooksAt(admin, "system", url);
|
||||||
|
if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) {
|
||||||
|
throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` +
|
||||||
|
`made (${ours.id})`);
|
||||||
|
}
|
||||||
|
const removed = { system: before.size, default: defaults.length, repository: copies.length,
|
||||||
|
repositories: [...new Set(copies.map((c) => c.repo))] };
|
||||||
|
return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [],
|
||||||
|
active: !!ours.active, content_type: ours.config?.content_type,
|
||||||
|
did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */
|
||||||
|
export function eventsTool(pace: Pace): { name: string; description: string; input: Record<string, unknown>;
|
||||||
|
run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> } {
|
||||||
|
return {
|
||||||
|
name: "gitea_events",
|
||||||
|
description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " +
|
||||||
|
"system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " +
|
||||||
|
"verified and the last one's time and event, how many were refused and the last reason, how often the polls look " +
|
||||||
|
"and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " +
|
||||||
|
"forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " +
|
||||||
|
"poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " +
|
||||||
|
"could not listen or the webhook could not be made.",
|
||||||
|
input: {},
|
||||||
|
run: async () => pace.report(),
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# kernel
|
||||||
|
|
||||||
|
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
|
||||||
|
**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
|
||||||
|
entries) is not built yet.
|
||||||
|
|
||||||
|
## Why (novox/hq issue 746)
|
||||||
|
|
||||||
|
With transparent huge pages `always`, the page cache of a file mapped by a program is collapsed into
|
||||||
|
2 MiB folios (khugepaged), and a small write into such a folio dirties all of it: writeback then writes
|
||||||
|
2 MiB. journald writes its journal files through mmap, and every journal reader (fail2ban's journal
|
||||||
|
jails among them) maps them too. Measured on the control node on 2026-10-14: journald's cgroup had
|
||||||
|
written 2.5 TB in 12 days while the journal grew by about 25-30 KB/s, fail2ban's cgroup 63 GB in 4 h
|
||||||
|
while its own files did not grow, both holding hundreds of MiB of file pages in huge folios; the bursts
|
||||||
|
saturated the mirrored spinning root disk that holds the bus store and the store, and the controller
|
||||||
|
lost its lease four times in 30 minutes. With `madvise`, only memory a program asks to have in huge pages
|
||||||
|
(`madvise(MADV_HUGEPAGE)`) gets them; a database or a JVM that wants them still asks.
|
||||||
|
|
||||||
|
## What it owns
|
||||||
|
|
||||||
|
| | what | notes |
|
||||||
|
|---|---|---|
|
||||||
|
| setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `madvise`, the value the mesh chose for every node (ADR 0381), not what the distribution's kernel boots with (`always`); a machine that needs another value is given it as its own setting, with its reason |
|
||||||
|
| file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` |
|
||||||
|
| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable |
|
||||||
|
| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's |
|
||||||
|
|
||||||
|
The per-size controls (`hugepages-<n>kB/enabled`) are left alone: on the machines read, 2048 kB says
|
||||||
|
`inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one.
|
||||||
|
|
||||||
|
**A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed
|
||||||
|
from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates
|
||||||
|
the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped
|
||||||
|
under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No
|
||||||
|
rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
|
||||||
|
or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| tool | | answers |
|
||||||
|
|---|---|---|
|
||||||
|
| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
|
||||||
|
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
|
||||||
|
writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
|
||||||
|
rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
|
||||||
|
reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
|
||||||
|
does not take before writing anything, fails when the write does not take, and says when the kernel has
|
||||||
|
no transparent huge pages.
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// THPDir is where the kernel says and takes its transparent huge pages.
|
||||||
|
const THPDir = "sys/kernel/mm/transparent_hugepage"
|
||||||
|
|
||||||
|
// CgroupRoot is the unified cgroup hierarchy.
|
||||||
|
const CgroupRoot = "sys/fs/cgroup"
|
||||||
|
|
||||||
|
// HugePagesAnswer is what kernel_huge_pages answers.
|
||||||
|
type HugePagesAnswer struct {
|
||||||
|
// Mode is the word the kernel brackets in enabled: always, madvise or never.
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
Defrag string `json:"defrag,omitempty"`
|
||||||
|
Shmem string `json:"shmem_enabled,omitempty"`
|
||||||
|
PerSize []Size `json:"per_size,omitempty"`
|
||||||
|
// Meminfo is the machine's huge pages by kind, in MiB.
|
||||||
|
Meminfo map[string]float64 `json:"meminfo_mib,omitempty"`
|
||||||
|
Units []Unit `json:"units"`
|
||||||
|
Said []string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size is one per-size control (hugepages-<n>kB/enabled).
|
||||||
|
type Size struct {
|
||||||
|
Size string `json:"size"`
|
||||||
|
Enabled string `json:"enabled"`
|
||||||
|
// Means is what inherit resolves to: the top-level mode.
|
||||||
|
Means string `json:"means"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unit is one cgroup holding file pages in huge folios.
|
||||||
|
type Unit struct {
|
||||||
|
Cgroup string `json:"cgroup"`
|
||||||
|
FileTHP int64 `json:"file_thp_bytes"`
|
||||||
|
FileDirty int64 `json:"file_dirty_bytes"`
|
||||||
|
FileWriteback int64 `json:"file_writeback_bytes"`
|
||||||
|
FileMapped int64 `json:"file_mapped_bytes"`
|
||||||
|
CollapsedByKhug int64 `json:"thp_collapse_alloc"`
|
||||||
|
Written int64 `json:"written_bytes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// bracketed is the word a sysfs choice file brackets: "always [madvise] never" is madvise.
|
||||||
|
func bracketed(text string) string {
|
||||||
|
open := strings.IndexByte(text, '[')
|
||||||
|
end := strings.IndexByte(text, ']')
|
||||||
|
if open < 0 || end < open {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return text[open+1 : end]
|
||||||
|
}
|
||||||
|
|
||||||
|
// HugePages reads the mode, the per-size controls, meminfo and the top units by file_thp.
|
||||||
|
func (m *Machine) HugePages(units int) (*HugePagesAnswer, error) {
|
||||||
|
enabled, err := m.read(THPDir + "/enabled")
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil, fmt.Errorf("this kernel has no /%s/enabled: it was built without transparent huge pages", THPDir)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("reading /%s/enabled: %w", THPDir, err)
|
||||||
|
}
|
||||||
|
a := &HugePagesAnswer{Mode: bracketed(enabled), Units: []Unit{}}
|
||||||
|
if s, err := m.read(THPDir + "/defrag"); err == nil {
|
||||||
|
a.Defrag = bracketed(s)
|
||||||
|
}
|
||||||
|
if s, err := m.read(THPDir + "/shmem_enabled"); err == nil {
|
||||||
|
a.Shmem = bracketed(s)
|
||||||
|
}
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("transparent huge pages: %s, defrag %s", a.Mode, orUnknown(a.Defrag)))
|
||||||
|
|
||||||
|
sizes, _ := m.dirs(THPDir)
|
||||||
|
for _, d := range sizes {
|
||||||
|
if !strings.HasPrefix(d, "hugepages-") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s, err := m.read(THPDir + "/" + d + "/enabled")
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sz := Size{Size: strings.TrimPrefix(d, "hugepages-"), Enabled: bracketed(s)}
|
||||||
|
sz.Means = sz.Enabled
|
||||||
|
if sz.Enabled == "inherit" {
|
||||||
|
sz.Means = a.Mode
|
||||||
|
}
|
||||||
|
a.PerSize = append(a.PerSize, sz)
|
||||||
|
}
|
||||||
|
sort.Slice(a.PerSize, func(i, j int) bool { return kib(a.PerSize[i].Size) < kib(a.PerSize[j].Size) })
|
||||||
|
var on []string
|
||||||
|
for _, s := range a.PerSize {
|
||||||
|
if s.Means != "never" {
|
||||||
|
on = append(on, s.Size+" "+s.Means)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(a.PerSize) > 0 {
|
||||||
|
a.Said = append(a.Said, "sizes not never: "+orNone(on))
|
||||||
|
}
|
||||||
|
|
||||||
|
if mi, err := m.read("proc/meminfo"); err == nil {
|
||||||
|
a.Meminfo = meminfo(mi)
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("huge pages held: anonymous %.0f MiB, shared memory %.0f MiB, file %.0f MiB (mapped at 2 MiB %.0f MiB)",
|
||||||
|
a.Meminfo["AnonHugePages"], a.Meminfo["ShmemHugePages"], a.Meminfo["FileHugePages"], a.Meminfo["FilePmdMapped"]))
|
||||||
|
}
|
||||||
|
|
||||||
|
if units > 0 {
|
||||||
|
all := m.cgroups()
|
||||||
|
sort.Slice(all, func(i, j int) bool {
|
||||||
|
if all[i].FileTHP != all[j].FileTHP {
|
||||||
|
return all[i].FileTHP > all[j].FileTHP
|
||||||
|
}
|
||||||
|
return all[i].Cgroup < all[j].Cgroup
|
||||||
|
})
|
||||||
|
if len(all) > units {
|
||||||
|
all = all[:units]
|
||||||
|
}
|
||||||
|
a.Units = all
|
||||||
|
for _, u := range all {
|
||||||
|
a.Said = append(a.Said, fmt.Sprintf("%s: file in huge folios %s, dirty %s, writeback %s, written %s since it started",
|
||||||
|
u.Cgroup, mib(u.FileTHP), mib(u.FileDirty), mib(u.FileWriteback), mib(u.Written)))
|
||||||
|
}
|
||||||
|
if len(all) == 0 {
|
||||||
|
a.Said = append(a.Said, "no unit holds file pages in huge folios")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return a, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cgroups walks the leaf services and scopes (to three levels below the root) that hold file_thp.
|
||||||
|
func (m *Machine) cgroups() []Unit {
|
||||||
|
var out []Unit
|
||||||
|
var walk func(rel string, depth int)
|
||||||
|
walk = func(rel string, depth int) {
|
||||||
|
children, _ := m.dirs(path.Join(CgroupRoot, rel))
|
||||||
|
for _, c := range children {
|
||||||
|
p := path.Join(rel, c)
|
||||||
|
if strings.HasSuffix(c, ".slice") {
|
||||||
|
if depth < 3 {
|
||||||
|
walk(p, depth+1)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasSuffix(c, ".service") && !strings.HasSuffix(c, ".scope") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stat, err := m.read(path.Join(CgroupRoot, p, "memory.stat"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kv := memoryStat(stat)
|
||||||
|
if kv["file_thp"] == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
u := Unit{Cgroup: p, FileTHP: kv["file_thp"], FileDirty: kv["file_dirty"], FileWriteback: kv["file_writeback"],
|
||||||
|
FileMapped: kv["file_mapped"], CollapsedByKhug: kv["thp_collapse_alloc"]}
|
||||||
|
if io, err := m.read(path.Join(CgroupRoot, p, "io.stat")); err == nil {
|
||||||
|
u.Written = mostWritten(io)
|
||||||
|
}
|
||||||
|
out = append(out, u)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk("", 0)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// memoryStat reads memory.stat's "key value" lines.
|
||||||
|
func memoryStat(text string) map[string]int64 {
|
||||||
|
kv := map[string]int64{}
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
f := strings.Fields(l)
|
||||||
|
if len(f) != 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n, err := strconv.ParseInt(f[1], 10, 64); err == nil {
|
||||||
|
kv[f[0]] = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kv
|
||||||
|
}
|
||||||
|
|
||||||
|
// mostWritten is the most bytes any one device of io.stat saw written: a mirror's member disks and the
|
||||||
|
// mirror itself each count the same write, so the lines are never summed.
|
||||||
|
func mostWritten(text string) int64 {
|
||||||
|
var most int64
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
for _, f := range strings.Fields(l) {
|
||||||
|
if v, ok := strings.CutPrefix(f, "wbytes="); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > most {
|
||||||
|
most = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return most
|
||||||
|
}
|
||||||
|
|
||||||
|
// meminfo reads the huge page lines of /proc/meminfo, in MiB.
|
||||||
|
func meminfo(text string) map[string]float64 {
|
||||||
|
want := map[string]bool{"AnonHugePages": true, "ShmemHugePages": true, "ShmemPmdMapped": true, "FileHugePages": true, "FilePmdMapped": true}
|
||||||
|
out := map[string]float64{}
|
||||||
|
for _, l := range strings.Split(text, "\n") {
|
||||||
|
k, v, ok := strings.Cut(l, ":")
|
||||||
|
if !ok || !want[k] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
f := strings.Fields(v)
|
||||||
|
if len(f) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n, err := strconv.ParseFloat(f[0], 64); err == nil {
|
||||||
|
out[k] = n / 1024
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func kib(size string) int {
|
||||||
|
n, _ := strconv.Atoi(strings.TrimSuffix(size, "kB"))
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }
|
||||||
|
|
||||||
|
func orUnknown(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(list []string) string {
|
||||||
|
if len(list) == 0 {
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
return strings.Join(list, ", ")
|
||||||
|
}
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for
|
||||||
|
// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and
|
||||||
|
// journald's memory.stat and io.stat, cut to the lines the tool reads.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func tree(t *testing.T, files map[string]string) *Machine {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
for p, content := range files {
|
||||||
|
full := filepath.Join(root, p)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &Machine{Root: root}
|
||||||
|
}
|
||||||
|
|
||||||
|
func novox(t *testing.T) *Machine {
|
||||||
|
thp := "sys/kernel/mm/transparent_hugepage/"
|
||||||
|
cg := "sys/fs/cgroup/system.slice/"
|
||||||
|
files := map[string]string{
|
||||||
|
thp + "enabled": "[always] madvise never\n",
|
||||||
|
thp + "defrag": "always defer defer+madvise [madvise] never\n",
|
||||||
|
thp + "shmem_enabled": "always within_size [advise] never deny force\n",
|
||||||
|
thp + "hugepages-2048kB/enabled": "always [inherit] madvise never\n",
|
||||||
|
thp + "hugepages-64kB/enabled": "always inherit madvise [never]\n",
|
||||||
|
thp + "hugepages-1024kB/enabled": "always inherit madvise [never]\n",
|
||||||
|
"proc/meminfo": "MemTotal: 65748808 kB\nAnonHugePages: 3579904 kB\nShmemHugePages: 4096 kB\n" +
|
||||||
|
"ShmemPmdMapped: 4096 kB\nFileHugePages: 1693696 kB\nFilePmdMapped: 882688 kB\nHugepagesize: 2048 kB\n",
|
||||||
|
cg + "fail2ban.service/memory.stat": "anon 21815296\nfile 204623872\nfile_mapped 130736128\nfile_dirty 27262976\n" +
|
||||||
|
"file_writeback 4096\nanon_thp 0\nfile_thp 197132288\nthp_fault_alloc 0\nthp_collapse_alloc 550\n",
|
||||||
|
cg + "fail2ban.service/io.stat": "9:0 \n8:16 rbytes=3223552 wbytes=67723415552 rios=240 wios=281159 dbytes=0 dios=0\n" +
|
||||||
|
"8:0 rbytes=6152192 wbytes=67723415552 rios=832 wios=281159 dbytes=0 dios=0\n" +
|
||||||
|
"9:2 rbytes=9375744 wbytes=67723415552 rios=1072 wios=281159 dbytes=0 dios=0\n",
|
||||||
|
cg + "systemd-journald.service/memory.stat": "anon 3227648\nfile 1032015872\nfile_mapped 131928064\nfile_dirty 622592\n" +
|
||||||
|
"file_writeback 159744\nanon_thp 2097152\nfile_thp 379584512\nthp_fault_alloc 0\nthp_collapse_alloc 11741\n",
|
||||||
|
cg + "systemd-journald.service/io.stat": "9:0 \n8:16 rbytes=55209984 wbytes=2498768035840 rios=1317 wios=24719004 dbytes=0 dios=0\n" +
|
||||||
|
"8:0 rbytes=72953856 wbytes=2498768039936 rios=2194 wios=24719005 dbytes=0 dios=0\n" +
|
||||||
|
"9:2 rbytes=128163840 wbytes=2498724843520 rios=3511 wios=24707873 dbytes=0 dios=0\n",
|
||||||
|
cg + "sshd.service/memory.stat": "anon 1000\nfile_thp 0\n",
|
||||||
|
}
|
||||||
|
return tree(t, files)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNovoxBeforeTheChange(t *testing.T) {
|
||||||
|
a, err := novox(t).HugePages(10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a.Mode != "always" || a.Defrag != "madvise" || a.Shmem != "advise" {
|
||||||
|
t.Fatalf("modes: %+v", a)
|
||||||
|
}
|
||||||
|
if len(a.PerSize) != 3 || a.PerSize[0].Size != "64kB" || a.PerSize[2].Size != "2048kB" ||
|
||||||
|
a.PerSize[2].Enabled != "inherit" || a.PerSize[2].Means != "always" || a.PerSize[0].Means != "never" {
|
||||||
|
t.Fatalf("per size: %+v", a.PerSize)
|
||||||
|
}
|
||||||
|
if a.Meminfo["FileHugePages"] != 1654 || a.Meminfo["FilePmdMapped"] != 862 {
|
||||||
|
t.Fatalf("meminfo: %v", a.Meminfo)
|
||||||
|
}
|
||||||
|
if len(a.Units) != 2 || a.Units[0].Cgroup != "system.slice/systemd-journald.service" ||
|
||||||
|
a.Units[1].Cgroup != "system.slice/fail2ban.service" {
|
||||||
|
t.Fatalf("units: %+v", a.Units)
|
||||||
|
}
|
||||||
|
// The mirror and its two members each count the write: the most any one saw, never their sum.
|
||||||
|
if a.Units[1].Written != 67723415552 || a.Units[1].FileTHP != 197132288 || a.Units[1].CollapsedByKhug != 550 {
|
||||||
|
t.Fatalf("fail2ban: %+v", a.Units[1])
|
||||||
|
}
|
||||||
|
said := strings.Join(a.Said, "\n")
|
||||||
|
for _, want := range []string{"transparent huge pages: always, defrag madvise", "sizes not never: 2048kB always",
|
||||||
|
"file 1654 MiB (mapped at 2 MiB 862 MiB)", "system.slice/fail2ban.service: file in huge folios 188.0 MiB"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Errorf("said lacks %q:\n%s", want, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) {
|
||||||
|
a, err := novox(t).HugePages(1)
|
||||||
|
if err != nil || len(a.Units) != 1 {
|
||||||
|
t.Fatalf("%v %+v", err, a)
|
||||||
|
}
|
||||||
|
b, err := tree(t, map[string]string{"sys/kernel/mm/transparent_hugepage/enabled": "always [madvise] never\n"}).HugePages(10)
|
||||||
|
if err != nil || b.Mode != "madvise" || len(b.Units) != 0 || !strings.Contains(strings.Join(b.Said, "\n"), "no unit holds") {
|
||||||
|
t.Fatalf("%v %+v", err, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKernelWithoutHugePagesIsAnError(t *testing.T) {
|
||||||
|
if _, err := tree(t, map[string]string{"proc/meminfo": ""}).HugePages(10); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "built without transparent huge pages") {
|
||||||
|
t.Fatalf("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheArgumentIsBounded(t *testing.T) {
|
||||||
|
if _, err := whole(map[string]any{"units": -1.0}, "units", 10, 0, 100); err == nil {
|
||||||
|
t.Error("a negative count was accepted")
|
||||||
|
}
|
||||||
|
if n, _ := whole(map[string]any{"units": 1000.0}, "units", 10, 0, 100); n != 100 {
|
||||||
|
t.Errorf("not bounded: %d", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The machine as the tools see it: the files they read, under a root that tests replace with a stand-in
|
||||||
|
// tree. Everything read here is readable by the runtime's account; nothing is escalated.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Machine reads this machine's kernel files under Root ("/" on the machine).
|
||||||
|
type Machine struct {
|
||||||
|
Root string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ThisMachine is the machine the runtime launched this bundle on.
|
||||||
|
func ThisMachine() *Machine { return &Machine{Root: "/"} }
|
||||||
|
|
||||||
|
func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) }
|
||||||
|
|
||||||
|
func (m *Machine) read(p string) (string, error) {
|
||||||
|
b, err := os.ReadFile(m.path(p))
|
||||||
|
return string(b), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Machine) dirs(p string) ([]string, error) {
|
||||||
|
entries, err := os.ReadDir(m.path(p))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || e.Type()&fs.ModeSymlink != 0 {
|
||||||
|
out = append(out, e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// whole is a whole-number argument with a default, kept within bounds.
|
||||||
|
func whole(args map[string]any, key string, def, least, most int) (int, error) {
|
||||||
|
raw, present := args[key]
|
||||||
|
if !present || raw == nil {
|
||||||
|
return def, nil
|
||||||
|
}
|
||||||
|
f, ok := raw.(float64)
|
||||||
|
if !ok || f != float64(int(f)) {
|
||||||
|
return 0, fmt.Errorf("%s must be a whole number", key)
|
||||||
|
}
|
||||||
|
n := int(f)
|
||||||
|
if n < least {
|
||||||
|
return 0, fmt.Errorf("%s must be at least %d", key, least)
|
||||||
|
}
|
||||||
|
if n > most {
|
||||||
|
n = most
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// schema is a tool's input: its properties.
|
||||||
|
func schema(properties map[string]any) map[string]any {
|
||||||
|
return map[string]any{"type": "object", "properties": properties}
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
// The kernel module's tools bundle (novox/hq to-be 42 Phase 1 row 11, issue 746).
|
||||||
|
//
|
||||||
|
// Served by the node's runtime over MCP on stdio through the Go SDK (ADR 0188, ADR 0193). It only reads:
|
||||||
|
// the transparent huge pages the kernel runs with, and which units hold file pages in huge folios, which
|
||||||
|
// is what a small write to a mapped file costs a whole huge page of writeback for (issue 746).
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// binaryName is what the build names this bundle's executable: the manifest's `binary`.
|
||||||
|
const binaryName = "kernel-tools"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): kernel.
|
||||||
|
if err := stdio.Serve("", tools(ThisMachine())); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func tools(m *Machine) []stdio.Tool {
|
||||||
|
return []stdio.Tool{
|
||||||
|
{
|
||||||
|
Name: "kernel_huge_pages",
|
||||||
|
Description: "Transparent huge pages on this machine: the mode in force (always, madvise or never) and defrag, " +
|
||||||
|
"each per-size control with what inherit resolves to, the machine's huge pages by kind (anonymous, " +
|
||||||
|
"shared memory, file, file mapped at 2 MiB) from /proc/meminfo, and the units holding the most file " +
|
||||||
|
"pages in huge folios (memory.stat file_thp) with their dirty and writeback bytes and the bytes their " +
|
||||||
|
"cgroup has written (io.stat, the most any one device saw). A unit with file_thp that writes far more " +
|
||||||
|
"than its files grow is paying a whole huge page of writeback per small write (novox/hq issue 746). " +
|
||||||
|
"Replaces cat /sys/kernel/mm/transparent_hugepage/enabled, grep Huge /proc/meminfo and reading " +
|
||||||
|
"memory.stat by hand. (r)",
|
||||||
|
Input: schema(map[string]any{
|
||||||
|
"units": map[string]any{"type": "integer", "description": "how many units to name, the most file_thp first (default 10, 0 for none, at most 100)"},
|
||||||
|
}),
|
||||||
|
Run: func(args map[string]any) (any, error) {
|
||||||
|
n, err := whole(args, "units", 10, 0, 100)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return m.HugePages(n)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,239 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The module's shape (novox/hq issue 746): the setting's default is madvise, the value the mesh chose
|
||||||
|
// (ADR 0381), not what the kernel boots with; the script is written into the
|
||||||
|
// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script,
|
||||||
|
// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies,
|
||||||
|
// gives back and refuses as it says.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
|
||||||
|
m := manifest(t)
|
||||||
|
s := m.Settings["transparent-huge-pages"]
|
||||||
|
if s["kind"] != "preference" || s["default"] != "madvise" || s["why"] == "" {
|
||||||
|
t.Fatalf("the setting: %v", s)
|
||||||
|
}
|
||||||
|
var ids []string
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
ids = append(ids, r["id"].(string))
|
||||||
|
}
|
||||||
|
if strings.Join(ids, " ") != "state huge-pages unit apply" {
|
||||||
|
t.Fatalf("order: %v", ids)
|
||||||
|
}
|
||||||
|
script := m.resource(t, "huge-pages")
|
||||||
|
// The script runs as root, so the setting it is written from is set at the terminal alone (issue 339).
|
||||||
|
if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true ||
|
||||||
|
!strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") {
|
||||||
|
t.Fatalf("the script: %v", script["path"])
|
||||||
|
}
|
||||||
|
unit := m.resource(t, "unit")["content"].(string)
|
||||||
|
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply",
|
||||||
|
"ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target",
|
||||||
|
"DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes",
|
||||||
|
"NoNewPrivileges=yes"} {
|
||||||
|
if !strings.Contains(unit, line+"\n") {
|
||||||
|
t.Errorf("the unit lacks %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
apply := m.resource(t, "apply")
|
||||||
|
if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
|
||||||
|
t.Fatalf("apply: %v", apply)
|
||||||
|
}
|
||||||
|
// A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the
|
||||||
|
// mode the machine was found with.
|
||||||
|
if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" {
|
||||||
|
t.Fatalf("restart-on: %v", on)
|
||||||
|
}
|
||||||
|
if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" {
|
||||||
|
t.Fatalf("reload-on: %v", on)
|
||||||
|
}
|
||||||
|
if h := apply["health"].(map[string]any); h["kind"] != "unit" {
|
||||||
|
t.Fatalf("health: %v", h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage
|
||||||
|
// directory, the module's state, and a PATH whose journalctl records that it was asked to rotate.
|
||||||
|
type bed struct {
|
||||||
|
script, sys, state, rotated, path, bin string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newBed(t *testing.T, want, found string) bed {
|
||||||
|
return newBedIn(t, t.TempDir(), want, found)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the
|
||||||
|
// state, as a reload with a new value does.
|
||||||
|
func newBedIn(t *testing.T, dir, want, found string) bed {
|
||||||
|
t.Helper()
|
||||||
|
b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"),
|
||||||
|
rotated: filepath.Join(dir, "rotated")}
|
||||||
|
content := manifest(t).resource(t, "huge-pages")["content"].(string)
|
||||||
|
content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want)
|
||||||
|
content = strings.ReplaceAll(content, "${dir:state}", b.state)
|
||||||
|
if strings.Contains(content, "${") {
|
||||||
|
t.Fatalf("a substitution the test does not make is left in the script")
|
||||||
|
}
|
||||||
|
bin := filepath.Join(dir, "bin")
|
||||||
|
for _, d := range []string{b.sys, b.state, bin} {
|
||||||
|
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
write(t, b.script, content, 0o755)
|
||||||
|
if found != "" {
|
||||||
|
write(t, filepath.Join(b.sys, "enabled"), found, 0o644)
|
||||||
|
}
|
||||||
|
write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755)
|
||||||
|
b.bin = bin
|
||||||
|
b.booted(t, "running")
|
||||||
|
b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH")
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// booted makes the service manager say state for is-system-running.
|
||||||
|
func (b bed) booted(t *testing.T, state string) {
|
||||||
|
t.Helper()
|
||||||
|
write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755)
|
||||||
|
}
|
||||||
|
|
||||||
|
func write(t *testing.T, p, content string, mode os.FileMode) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(p, []byte(content), mode); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bed) run(t *testing.T, verb string) (string, error) {
|
||||||
|
t.Helper()
|
||||||
|
cmd := exec.Command("sh", b.script, verb, b.sys, b.state)
|
||||||
|
cmd.Env = append(os.Environ(), "PATH="+b.path)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
return string(out), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bed) file(t *testing.T, p string) string {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(p)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) {
|
||||||
|
// novox's file on 2026-10-14, read through node-reads.
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "always -> madvise") {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
|
||||||
|
t.Fatalf("written: %q", got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.state, "found")); got != "always\n" {
|
||||||
|
t.Fatalf("found: %q", got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "--rotate\n" {
|
||||||
|
t.Fatalf("rotation: %q", got)
|
||||||
|
}
|
||||||
|
// Applied again (a restart with the same value): nothing written, nothing rotated, found kept.
|
||||||
|
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise, as set") {
|
||||||
|
t.Fatalf("again: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "--rotate\n" {
|
||||||
|
t.Fatalf("rotated again: %q", got)
|
||||||
|
}
|
||||||
|
if out, err := b.run(t, "restore"); err != nil || !strings.Contains(out, "back to always") {
|
||||||
|
t.Fatalf("restore: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" {
|
||||||
|
t.Fatalf("restored: %q", got)
|
||||||
|
}
|
||||||
|
// Nothing of the module is kept once it is stopped.
|
||||||
|
if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("found kept after restore: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
b := newBedIn(t, dir, "madvise", "[always] madvise never\n")
|
||||||
|
if out, err := b.run(t, "apply"); err != nil {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
// The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again.
|
||||||
|
n := newBedIn(t, dir, "never", "")
|
||||||
|
out, err := n.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "madvise -> never") {
|
||||||
|
t.Fatalf("reload: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" {
|
||||||
|
t.Fatalf("found rewritten by a reload: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoRotationWhileTheMachineIsStarting(t *testing.T) {
|
||||||
|
for _, state := range []string{"initializing", "starting"} {
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
b.booted(t, state)
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err != nil || !strings.Contains(out, "still starting") {
|
||||||
|
t.Fatalf("%s: %v %s", state, err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
|
||||||
|
t.Fatalf("%s: not set: %q", state, got)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "" {
|
||||||
|
t.Fatalf("%s: rotated at boot: %q", state, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAlwaysIsSetWithoutRotating(t *testing.T) {
|
||||||
|
b := newBed(t, "always", "always [madvise] never\n")
|
||||||
|
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise -> always") {
|
||||||
|
t.Fatalf("apply: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, b.rotated); got != "" {
|
||||||
|
t.Fatalf("rotated for always: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAValueTheKernelDoesNotTakeIsRefusedBeforeAnythingIsWritten(t *testing.T) {
|
||||||
|
b := newBed(t, "sometimes", "[always] madvise never\n")
|
||||||
|
out, err := b.run(t, "apply")
|
||||||
|
if err == nil || !strings.Contains(out, "takes always, madvise or never") {
|
||||||
|
t.Fatalf("accepted: %v %s", err, out)
|
||||||
|
}
|
||||||
|
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "[always] madvise never\n" {
|
||||||
|
t.Fatalf("written anyway: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) {
|
||||||
|
if os.Getuid() == 0 {
|
||||||
|
t.Skip("root writes a read-only file")
|
||||||
|
}
|
||||||
|
b := newBed(t, "madvise", "[always] madvise never\n")
|
||||||
|
if err := os.Chmod(filepath.Join(b.sys, "enabled"), 0o444); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if out, err := b.run(t, "apply"); err == nil {
|
||||||
|
t.Fatalf("a refused write passed: %s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAKernelWithoutHugePagesIsSaid(t *testing.T) {
|
||||||
|
b := newBed(t, "madvise", "")
|
||||||
|
if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") {
|
||||||
|
t.Fatalf("%v %s", err, out)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type resource map[string]any
|
||||||
|
|
||||||
|
type manifestShape struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Capabilities []string `json:"capabilities"`
|
||||||
|
Settings map[string]map[string]any `json:"settings"`
|
||||||
|
Tools []string `json:"tools"`
|
||||||
|
Resources []resource `json:"resources"`
|
||||||
|
Build struct {
|
||||||
|
Artifacts []map[string]any `json:"artifacts"`
|
||||||
|
} `json:"build"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func manifest(t *testing.T) manifestShape {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m manifestShape
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m manifestShape) resource(t *testing.T, id string) resource {
|
||||||
|
t.Helper()
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["id"] == id {
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no resource %s", id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestToolsAreTheManifests holds the served tools and the manifest's list to one another, and the
|
||||||
|
// bundle to the shape the builder compiles and the runtime loads.
|
||||||
|
func TestToolsAreTheManifests(t *testing.T) {
|
||||||
|
m := manifest(t)
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, tool := range tools(&Machine{Root: t.TempDir()}) {
|
||||||
|
if names[tool.Name] {
|
||||||
|
t.Errorf("%s is served twice", tool.Name)
|
||||||
|
}
|
||||||
|
names[tool.Name] = true
|
||||||
|
}
|
||||||
|
for _, want := range m.Tools {
|
||||||
|
if !names[want] {
|
||||||
|
t.Errorf("the manifest lists %s and the bundle does not serve it", want)
|
||||||
|
}
|
||||||
|
delete(names, want)
|
||||||
|
}
|
||||||
|
if len(names) != 0 {
|
||||||
|
t.Errorf("served and not listed: %v", names)
|
||||||
|
}
|
||||||
|
var tools map[string]any
|
||||||
|
for _, a := range m.Build.Artifacts {
|
||||||
|
if a["name"] == "tools" {
|
||||||
|
tools = a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tools == nil || tools["kind"] != "bundle" || tools["language"] != "go" || tools["system"] != "arch" ||
|
||||||
|
tools["from"] != "cmd/"+binaryName || tools["binary"] != binaryName {
|
||||||
|
t.Fatalf("the tools artifact: %v", tools)
|
||||||
|
}
|
||||||
|
if loads, _ := tools["loads"].([]any); len(loads) != 1 || loads[0] != binaryName {
|
||||||
|
t.Fatalf("loads: %v", tools["loads"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module kernel
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require git.novox.be/novox/mesh-sdk/go v0.1.7
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"module": "kernel",
|
||||||
|
"version": "1",
|
||||||
|
"capabilities": [
|
||||||
|
"service-manager"
|
||||||
|
],
|
||||||
|
"settings": {
|
||||||
|
"transparent-huge-pages": {
|
||||||
|
"kind": "preference",
|
||||||
|
"default": "madvise",
|
||||||
|
"why": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"tools": [
|
||||||
|
"kernel_huge_pages"
|
||||||
|
],
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "state",
|
||||||
|
"type": "directory",
|
||||||
|
"mode": "0700",
|
||||||
|
"place": "."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "huge-pages",
|
||||||
|
"type": "file",
|
||||||
|
"path": "${dir:state}/huge-pages",
|
||||||
|
"mode": "0755",
|
||||||
|
"trusted": true,
|
||||||
|
"content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [<the kernel's transparent_hugepage directory> [<this module's state>]]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [<transparent_hugepage directory> [<state directory>]]\" >&2\n\texit 2\n\t;;\nesac\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "unit",
|
||||||
|
"type": "file",
|
||||||
|
"path": "/etc/systemd/system/mesh-kernel-huge-pages.service",
|
||||||
|
"mode": "0644",
|
||||||
|
"content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "apply",
|
||||||
|
"type": "service",
|
||||||
|
"unit": "mesh-kernel-huge-pages.service",
|
||||||
|
"state": "running",
|
||||||
|
"boot": "enabled",
|
||||||
|
"restart-on": [
|
||||||
|
"unit"
|
||||||
|
],
|
||||||
|
"reload-on": [
|
||||||
|
"huge-pages"
|
||||||
|
],
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/kernel-tools",
|
||||||
|
"binary": "kernel-tools",
|
||||||
|
"loads": [
|
||||||
|
"kernel-tools"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one.
|
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
|
||||||
|
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
|
||||||
|
// past the grace without its word it is listed by `stalled`, saying why it was not given.
|
||||||
func (h *Holder) heldWalk(w Walk) bool {
|
func (h *Holder) heldWalk(w Walk) bool {
|
||||||
for _, d := range h.deliveries {
|
for _, d := range h.deliveries {
|
||||||
if d.State.Final() {
|
if d.State.Final() {
|
||||||
@@ -292,12 +294,28 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
w := s.walk
|
w := s.walk
|
||||||
|
if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 {
|
||||||
|
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
|
||||||
|
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
|
||||||
|
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
|
||||||
|
if why := h.lapsedSaid(w); why != "" {
|
||||||
|
says += ": " + why
|
||||||
|
}
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
|
||||||
|
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
||||||
|
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
|
||||||
|
continue
|
||||||
|
}
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
|
||||||
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
|
||||||
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
|
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
|
||||||
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
|
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
|
||||||
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
|
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
|
||||||
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
|
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
|
||||||
|
out[len(out)-1].Says += h.endedSaid(w)
|
||||||
|
if hold := h.coverHeld(id); hold != "" {
|
||||||
|
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@@ -344,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) {
|
|||||||
}
|
}
|
||||||
case !w.Open() && now.Sub(s.at) > keptWalkFor:
|
case !w.Open() && now.Sub(s.at) > keptWalkFor:
|
||||||
delete(h.walks, id)
|
delete(h.walks, id)
|
||||||
|
delete(h.deliverErr, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -356,6 +375,168 @@ type turnWant struct {
|
|||||||
whys []string
|
whys []string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
|
||||||
|
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
|
||||||
|
func (h *Holder) endedSaid(w Walk) string {
|
||||||
|
var ended []string
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if !d.State.Final() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, answers := walkAnswers(w, d)
|
||||||
|
if answers || d.Walk != nil && d.Walk.ID == w.ID {
|
||||||
|
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ended) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
sort.Strings(ended)
|
||||||
|
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
|
||||||
|
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
|
||||||
|
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
|
||||||
|
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
|
||||||
|
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
|
||||||
|
// itself, or by a person, gets nothing from it. Under the lock.
|
||||||
|
//
|
||||||
|
// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached
|
||||||
|
// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in
|
||||||
|
// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives
|
||||||
|
// nothing; lapsedBehind names that one.
|
||||||
|
func (h *Holder) lapsedOn(w Walk) []string {
|
||||||
|
out, _ := h.lapsedOf(w)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that
|
||||||
|
// lapsed whose merge the walk answers. Under the lock.
|
||||||
|
func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) {
|
||||||
|
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
behind := map[string]string{}
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if !h.lapsedAnswered(w, d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if before := h.lapsedBehind(w, d); before != "" {
|
||||||
|
behind[d.ID] = before
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, d.ID)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, behind
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
|
||||||
|
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
|
||||||
|
if d.MergedAs == "" || !h.stopLapsed(d) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, ok := walkAnswers(w, d); !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, left := w.leftOut(d.Repository, d.MergedAs)
|
||||||
|
return !left
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this
|
||||||
|
// walk does not deliver with it, with its state; empty for none.
|
||||||
|
func (h *Holder) lapsedBehind(w Walk, d *Delivery) string {
|
||||||
|
g := h.groups[d.Group]
|
||||||
|
if d.Group == "" || g == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
members := h.membersOf(g)
|
||||||
|
byID := map[string]*Delivery{}
|
||||||
|
for _, m := range members {
|
||||||
|
byID[m.ID] = m
|
||||||
|
}
|
||||||
|
for _, m := range h.inOrder(g, members) {
|
||||||
|
if m.ID == d.ID {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
o := byID[m.ID]
|
||||||
|
if o == nil || memberReached(o) || h.lapsedAnswered(w, o) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s)", o.ID, o.State)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
|
||||||
|
// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock.
|
||||||
|
func (h *Holder) lapsedSaid(w Walk) string {
|
||||||
|
var parts []string
|
||||||
|
_, behind := h.lapsedOf(w)
|
||||||
|
ids := make([]string, 0, len(behind))
|
||||||
|
for id := range behind {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
sort.Strings(ids)
|
||||||
|
for _, id := range ids {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id]))
|
||||||
|
}
|
||||||
|
if why := h.waitsWhy[w.ID]; why != "" {
|
||||||
|
parts = append(parts, "not let go yet: "+why)
|
||||||
|
}
|
||||||
|
if err := h.deliverErr[w.ID]; err != "" {
|
||||||
|
parts = append(parts, "the controller refused its word: "+err)
|
||||||
|
}
|
||||||
|
lapsed, _ := h.lapsedOf(w)
|
||||||
|
for _, id := range lapsed {
|
||||||
|
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
|
||||||
|
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
|
||||||
|
id, d.LetGo.Local().Format("15:04:05")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(parts, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered.
|
||||||
|
func (h *Holder) lapsedOwed(w Walk) bool {
|
||||||
|
lapsed, behind := h.lapsedOf(w)
|
||||||
|
return len(lapsed) > 0 || len(behind) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
|
||||||
|
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
|
||||||
|
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
|
||||||
|
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
|
||||||
|
func (h *Holder) lapsedTurns() {
|
||||||
|
for id, s := range h.walks {
|
||||||
|
for _, did := range h.lapsedOn(s.walk) {
|
||||||
|
d := h.deliveries[did]
|
||||||
|
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.turning == nil {
|
||||||
|
h.turning = map[string]*turnWant{}
|
||||||
|
}
|
||||||
|
want := h.turning[id]
|
||||||
|
if want == nil {
|
||||||
|
want = &turnWant{}
|
||||||
|
h.turning[id] = want
|
||||||
|
}
|
||||||
|
if !containsString(want.ids, d.ID) {
|
||||||
|
want.ids = append(want.ids, d.ID)
|
||||||
|
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
|
||||||
|
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
|
||||||
|
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
|
||||||
|
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
|
||||||
|
|
||||||
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
|
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
|
||||||
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
|
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
|
||||||
// Under the lock.
|
// Under the lock.
|
||||||
@@ -388,6 +569,9 @@ func (h *Holder) words() []func() {
|
|||||||
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
|
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
|
||||||
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
|
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
|
||||||
"asked to leave that merge out first"
|
"asked to leave that merge out first"
|
||||||
|
if hold := h.coverHeld(walk); hold != "" {
|
||||||
|
h.waitsWhy[walk] = hold
|
||||||
|
}
|
||||||
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
|
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
|
||||||
strings.Join(stopped, ", "))
|
strings.Join(stopped, ", "))
|
||||||
continue
|
continue
|
||||||
@@ -403,9 +587,14 @@ func (h *Holder) words() []func() {
|
|||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
delete(h.asking, walk)
|
delete(h.asking, walk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if h.deliverErr == nil {
|
||||||
|
h.deliverErr = map[string]string{}
|
||||||
|
}
|
||||||
|
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
|
||||||
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
|
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
delete(h.deliverErr, walk)
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
d := h.deliveries[id]
|
d := h.deliveries[id]
|
||||||
if d == nil {
|
if d == nil {
|
||||||
@@ -479,9 +668,28 @@ func (h *Holder) WalksWaiting() []StalledLine {
|
|||||||
switch {
|
switch {
|
||||||
case len(held) > 0:
|
case len(held) > 0:
|
||||||
out = append(out, held...)
|
out = append(out, held...)
|
||||||
|
case h.coverHeld(id) != "":
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
|
Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound",
|
||||||
|
Says: "held back: " + h.coverHeld(id)})
|
||||||
case h.waitsWhy[id] != "":
|
case h.waitsWhy[id] != "":
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
|
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
|
||||||
|
case !live && h.lapsedOwed(w):
|
||||||
|
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
|
||||||
|
// unheld, while the controller has not taken it, with what holds it.
|
||||||
|
lapsed, _ := h.lapsedOf(w)
|
||||||
|
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
|
||||||
|
"delivered since (novox/hq issue 753)"
|
||||||
|
if len(lapsed) > 0 {
|
||||||
|
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
|
||||||
|
"whose merge it answers (novox/hq issue 753)"
|
||||||
|
}
|
||||||
|
if why := h.lapsedSaid(w); why != "" {
|
||||||
|
says += "; " + why
|
||||||
|
}
|
||||||
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
|
||||||
|
Bound: "0s", H2: "none: it goes at its turn", Says: says})
|
||||||
case !live:
|
case !live:
|
||||||
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
|
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
|
||||||
"`mesh-delivery.show` for the deliveries of " + w.Repository
|
"`mesh-delivery.show` for the deliveries of " + w.Repository
|
||||||
@@ -492,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine {
|
|||||||
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
|
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
|
||||||
}
|
}
|
||||||
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
|
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
|
||||||
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says})
|
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool {
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
|||||||
@@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) {
|
|||||||
case Delivered:
|
case Delivered:
|
||||||
return "success", "delivered"
|
return "success", "delivered"
|
||||||
case Failed:
|
case Failed:
|
||||||
|
if d.coveredStop() {
|
||||||
|
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747)
|
||||||
|
}
|
||||||
return "failure", clip("failed: "+why, 140)
|
return "failure", clip("failed: "+why, 140)
|
||||||
case Superseded:
|
case Superseded:
|
||||||
return "warning", clip("superseded: "+why, 140)
|
return "warning", clip("superseded: "+why, 140)
|
||||||
@@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string {
|
|||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
|
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
|
||||||
if d.coveredStop() {
|
if d.coveredStop() {
|
||||||
fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy)
|
fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid())
|
||||||
}
|
}
|
||||||
if d.HeldWhy != "" && d.State == Held {
|
if d.HeldWhy != "" && d.State == Held {
|
||||||
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
|
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
|
||||||
|
|||||||
@@ -106,6 +106,8 @@ type fakeController struct {
|
|||||||
moves map[string][]string
|
moves map[string][]string
|
||||||
plans int
|
plans int
|
||||||
down bool
|
down bool
|
||||||
|
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
|
||||||
|
refuseDeliver bool
|
||||||
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
|
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
|
||||||
onCheck func()
|
onCheck func()
|
||||||
// planned is what Plan answers instead of "builds app", when set.
|
// planned is what Plan answers instead of "builds app", when set.
|
||||||
@@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error {
|
|||||||
if c.down {
|
if c.down {
|
||||||
return errors.New("down")
|
return errors.New("down")
|
||||||
}
|
}
|
||||||
|
if c.refuseDeliver {
|
||||||
|
return errors.New("the controller refuses the word")
|
||||||
|
}
|
||||||
w, known := c.walks[walk]
|
w, known := c.walks[walk]
|
||||||
if !known || w.Delivery == nil || w.Delivery.Go != nil {
|
if !known || w.Delivery == nil || w.Delivery.Go != nil {
|
||||||
return fmt.Errorf("%s waits for nobody", walk)
|
return fmt.Errorf("%s waits for nobody", walk)
|
||||||
|
|||||||
@@ -60,6 +60,8 @@ type Holder struct {
|
|||||||
waitsWhy map[string]string
|
waitsWhy map[string]string
|
||||||
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
|
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
|
||||||
asking map[string]bool
|
asking map[string]bool
|
||||||
|
// deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753).
|
||||||
|
deliverErr map[string]string
|
||||||
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
|
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
|
||||||
leaving map[string][]string
|
leaving map[string][]string
|
||||||
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
|
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
|
||||||
@@ -67,6 +69,13 @@ type Holder struct {
|
|||||||
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
|
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
|
||||||
// to stop the walk whole.
|
// to stop the walk whole.
|
||||||
leaveRefused map[string]string
|
leaveRefused map[string]string
|
||||||
|
// coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the
|
||||||
|
// forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of
|
||||||
|
// them is held back meanwhile, never asked to leave it out before the answer.
|
||||||
|
coverAsked map[string]bool
|
||||||
|
// coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past
|
||||||
|
// coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747).
|
||||||
|
coverAskedSince map[string]time.Time
|
||||||
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
|
||||||
tried owedTries
|
tried owedTries
|
||||||
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
|
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
|
||||||
@@ -303,19 +312,25 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
id := IDOf(p.repository(), p.HeadSHA)
|
id := IDOf(p.repository(), p.HeadSHA)
|
||||||
now := h.Now()
|
now := h.Now()
|
||||||
|
startedOver := ""
|
||||||
if d := h.deliveries[id]; d != nil {
|
if d := h.deliveries[id]; d != nil {
|
||||||
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
|
if startedOver = startedOverFor(d, p.Number); startedOver == "" {
|
||||||
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
|
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
|
||||||
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
|
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
|
||||||
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
|
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
|
||||||
d.BaseSHA = p.BaseSHA
|
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
|
||||||
h.keep(d)
|
d.BaseSHA = p.BaseSHA
|
||||||
|
h.keep(d)
|
||||||
|
}
|
||||||
|
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
|
||||||
|
d.AskedAgain = &now
|
||||||
|
h.keep(d)
|
||||||
|
}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
|
// A delivery stopped only because its pull request closed unmerged does not end its commit's delivery for
|
||||||
d.AskedAgain = &now
|
// another pull request with the same head (novox/hq issue 783): started over below, exactly as for a first
|
||||||
h.keep(d)
|
// announcement, under the same id — never a transition out of Stopped.
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
replaced := ""
|
replaced := ""
|
||||||
for _, old := range h.deliveries {
|
for _, old := range h.deliveries {
|
||||||
@@ -329,11 +344,7 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title,
|
d, t, err := h.freshAnnouncement(p, now, startedOver)
|
||||||
Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
|
|
||||||
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
|
|
||||||
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
|
|
||||||
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.refuse(err)
|
h.refuse(err)
|
||||||
return
|
return
|
||||||
@@ -351,6 +362,34 @@ func (h *Holder) PullUpdated(p PullEvent) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// startedOverFor is why a delivery already kept at a head's id is started over for a pull request's number rather
|
||||||
|
// than taken as announced again: only when it was stopped solely by an earlier, closed duplicate pull request with
|
||||||
|
// the same head — still Stopped, ClosedUnmerged, never merged, and under another number (novox/hq issue 783).
|
||||||
|
// Empty for the ordinary 'announced again' path, or for any other stop (a person's, a group's, its own close).
|
||||||
|
func startedOverFor(d *Delivery, number int) string {
|
||||||
|
if d != nil && d.State == Stopped && d.ClosedUnmerged && d.MergedAs == "" && d.Number != number {
|
||||||
|
return fmt.Sprintf("#%d with the same head closed unmerged", d.Number)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// freshAnnouncement makes a delivery for p's head exactly as a first announcement would: its own id, proposed.
|
||||||
|
// startedOver, when not empty, is folded into the transition's Why, naming the pull request this replaces
|
||||||
|
// (novox/hq issue 783); the delivery it replaces is swapped out by the caller keeping the new one under the same
|
||||||
|
// id (keep), never by a transition out of its final state.
|
||||||
|
func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver string) (*Delivery, Transition, error) {
|
||||||
|
d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number,
|
||||||
|
Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
|
||||||
|
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
|
||||||
|
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
|
||||||
|
why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)
|
||||||
|
if startedOver != "" {
|
||||||
|
why = fmt.Sprintf("%s; %s", why, startedOver)
|
||||||
|
}
|
||||||
|
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: why})
|
||||||
|
return d, t, err
|
||||||
|
}
|
||||||
|
|
||||||
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
|
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
|
||||||
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
|
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
|
||||||
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
|
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the
|
||||||
@@ -482,7 +521,29 @@ func (h *Holder) PullMerged(p PullEvent) {
|
|||||||
}
|
}
|
||||||
now := h.Now()
|
now := h.Now()
|
||||||
var ts []Transition
|
var ts []Transition
|
||||||
if d == nil {
|
if over := startedOverFor(d, p.Number); over != "" {
|
||||||
|
// A delivery stopped only by an earlier, closed duplicate pull request with the same head is not lost when
|
||||||
|
// its merge is heard without the open announcement either: it is started over for the merging pull request
|
||||||
|
// the same way an announcement would, before the merge is taken as usual (novox/hq issue 783). Heard this
|
||||||
|
// way — its own merge, with no open announcement of this pull request — it is made exactly as a first-heard
|
||||||
|
// merge in every field but its id: FromForge, and its check the forge's word on the head, so a long-past
|
||||||
|
// merge replayed from the bus's history still falls under the history rule (issue 313) and is not held for
|
||||||
|
// a person (review round 1).
|
||||||
|
fresh, t, err := h.freshAnnouncement(p, now, over)
|
||||||
|
if err != nil {
|
||||||
|
h.refuse(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fresh.FromForge = true
|
||||||
|
if s := p.HeadChecks["mesh/merge-gate"]; s != "" {
|
||||||
|
fresh.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it",
|
||||||
|
Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])}
|
||||||
|
}
|
||||||
|
d, ts = fresh, []Transition{t}
|
||||||
|
if d.Check != nil {
|
||||||
|
ts = append(ts, settle(d, Facts{Now: now})...)
|
||||||
|
}
|
||||||
|
} else if d == nil {
|
||||||
head := p.HeadSHA
|
head := p.HeadSHA
|
||||||
if head == "" {
|
if head == "" {
|
||||||
head = p.MergeCommit
|
head = p.MergeCommit
|
||||||
@@ -1112,6 +1173,48 @@ func (h *Holder) plan() []func() {
|
|||||||
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
|
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
|
||||||
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
|
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
|
||||||
// grace, it is stopped saying so — never left waiting for a start that cannot come.
|
// grace, it is stopped saying so — never left waiting for a start that cannot come.
|
||||||
|
// A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its
|
||||||
|
// change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it
|
||||||
|
// (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this
|
||||||
|
// was built clears by itself after a restart.
|
||||||
|
h.coverAsked = map[string]bool{}
|
||||||
|
if h.coverAskedSince == nil {
|
||||||
|
h.coverAskedSince = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
pending := map[string]bool{}
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if d.State != Failed && d.State != Stopped || d.coveredStop() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
by, ask := h.walkCover(d)
|
||||||
|
if len(ask) > 0 && by == "" {
|
||||||
|
pending[d.ID] = true
|
||||||
|
since, asked := h.coverAskedSince[d.ID]
|
||||||
|
if !asked {
|
||||||
|
since = now
|
||||||
|
h.coverAskedSince[d.ID] = now
|
||||||
|
}
|
||||||
|
// Held back only within the bound: a forge that keeps erroring never holds a walk for good.
|
||||||
|
h.coverAsked[d.ID] = now.Sub(since) < coverAskBound
|
||||||
|
}
|
||||||
|
if by != "" {
|
||||||
|
d.DeliveredBy = by
|
||||||
|
h.tellCovered(d, now)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, in := range ask {
|
||||||
|
if a := h.askContains(d, in, now); a != nil {
|
||||||
|
asks = append(asks, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for id := range h.coverAskedSince {
|
||||||
|
if !pending[id] {
|
||||||
|
delete(h.coverAskedSince, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, d := range h.deliveries {
|
for _, d := range h.deliveries {
|
||||||
ended := d.endedUnstarted()
|
ended := d.endedUnstarted()
|
||||||
if d.coveredStop() && !d.CoverTold {
|
if d.coveredStop() && !d.CoverTold {
|
||||||
@@ -1195,6 +1298,8 @@ func (h *Holder) plan() []func() {
|
|||||||
}
|
}
|
||||||
h.turn(d, "its turn: a delivery in no group")
|
h.turn(d, "its turn: a delivery in no group")
|
||||||
}
|
}
|
||||||
|
// A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753).
|
||||||
|
h.lapsedTurns()
|
||||||
// The word, once per walk, when every live delivery it carries is at its turn.
|
// The word, once per walk, when every live delivery it carries is at its turn.
|
||||||
asks = append(asks, h.words()...)
|
asks = append(asks, h.words()...)
|
||||||
|
|
||||||
@@ -1273,7 +1378,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string {
|
|||||||
// stays final, and the bus's transition event is not said again.
|
// stays final, and the bus's transition event is not said again.
|
||||||
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
||||||
d.CoverTold = true
|
d.CoverTold = true
|
||||||
line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid()))
|
line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid()))
|
||||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
|
||||||
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
|
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
|
||||||
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
|
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
|
||||||
@@ -1283,7 +1388,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) {
|
|||||||
state, desc := statusOf(d)
|
state, desc := statusOf(d)
|
||||||
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
|
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
|
||||||
}
|
}
|
||||||
h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid())
|
h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid())
|
||||||
h.keep(d)
|
h.keep(d)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1294,6 +1399,104 @@ func strandedUncovered(d *Delivery) bool {
|
|||||||
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
|
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or
|
||||||
|
// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it
|
||||||
|
// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or
|
||||||
|
// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not
|
||||||
|
// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every
|
||||||
|
// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own
|
||||||
|
// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its
|
||||||
|
// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records
|
||||||
|
// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart.
|
||||||
|
func (h *Holder) walkCover(d *Delivery) (string, []string) {
|
||||||
|
if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
var ask []string
|
||||||
|
for _, c := range h.doneWalksOf(d.Repository) {
|
||||||
|
w := c.walk
|
||||||
|
if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var missing []string
|
||||||
|
for _, m := range d.Plan.Moved {
|
||||||
|
unchanged := false
|
||||||
|
if d.Walk != nil {
|
||||||
|
_, unchanged = d.Walk.Unchanged[m]
|
||||||
|
}
|
||||||
|
if !containsString(w.Moved, m) && !unchanged {
|
||||||
|
missing = append(missing, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(missing) > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !sameCommit(c.commit, d.MergedAs) {
|
||||||
|
yes, known := d.ContainedIn[c.commit]
|
||||||
|
if !known && !containsString(ask, c.commit) {
|
||||||
|
ask = append(ask, c.commit)
|
||||||
|
}
|
||||||
|
if !yes {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8),
|
||||||
|
orNone(d.Base), shortOf(d.MergedAs, 8)), nil
|
||||||
|
}
|
||||||
|
return "", ask
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneWalk is a finished walk and the commit of its repository it carried.
|
||||||
|
type doneWalk struct {
|
||||||
|
walk *WalkSeen
|
||||||
|
commit string
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it
|
||||||
|
// holds, and each delivery's kept walk with the commit of its repository that walk carried for it.
|
||||||
|
func (h *Holder) doneWalksOf(repository string) []doneWalk {
|
||||||
|
byID := map[string]doneWalk{}
|
||||||
|
for id, s := range h.walks {
|
||||||
|
w := s.walk
|
||||||
|
if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit}
|
||||||
|
}
|
||||||
|
for _, o := range h.deliveries {
|
||||||
|
if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, known := byID[o.Walk.ID]; known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
commit := o.AnsweredBy
|
||||||
|
if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) {
|
||||||
|
commit = o.MergedAs
|
||||||
|
}
|
||||||
|
if commit == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit}
|
||||||
|
}
|
||||||
|
out := make([]doneWalk, 0, len(byID))
|
||||||
|
for _, c := range byID {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if !out[i].walk.Created.Equal(out[j].walk.Created) {
|
||||||
|
return out[i].walk.Created.Before(out[j].walk.Created)
|
||||||
|
}
|
||||||
|
return out[i].walk.ID < out[j].walk.ID
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed).
|
||||||
|
func (h *Holder) stopLapsed(d *Delivery) bool {
|
||||||
|
return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] })
|
||||||
|
}
|
||||||
|
|
||||||
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
|
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
|
||||||
// again after it: what holds it for a person, unless something already ends it.
|
// again after it: what holds it for a person, unless something already ends it.
|
||||||
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
|
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
|
||||||
@@ -1736,7 +1939,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
|
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
|
||||||
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
|
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
|
||||||
// issues 442 and 445).
|
// issues 442 and 445).
|
||||||
if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" {
|
// A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747).
|
||||||
|
if memberReached(d) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
|
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
|
||||||
@@ -1761,7 +1965,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
}
|
}
|
||||||
if w.why != "" {
|
if w.why != "" {
|
||||||
for _, later := range ordered[i+1:] {
|
for _, later := range ordered[i+1:] {
|
||||||
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
|
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
|
||||||
d.ID, w.why)); a != nil {
|
d.ID, w.why)); a != nil {
|
||||||
asks = append(asks, a)
|
asks = append(asks, a)
|
||||||
}
|
}
|
||||||
@@ -1788,9 +1992,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
return asks
|
return asks
|
||||||
|
case h.stopLapsed(d):
|
||||||
|
// Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is
|
||||||
|
// delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747).
|
||||||
|
continue
|
||||||
case d.State == Failed || d.State == Stopped:
|
case d.State == Failed || d.State == Stopped:
|
||||||
for _, later := range ordered[i+1:] {
|
for _, later := range ordered[i+1:] {
|
||||||
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
|
if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
|
||||||
asks = append(asks, a)
|
asks = append(asks, a)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2028,7 +2236,7 @@ func mayCarry(c, d *Delivery, module string) bool {
|
|||||||
// controller ends the walk and walks the other merges again without it, and without each later merge of its
|
// controller ends the walk and walks the other merges again without it, and without each later merge of its
|
||||||
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
|
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
|
||||||
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
|
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
|
||||||
func (h *Holder) stopInGroup(d *Delivery, why string) func() {
|
func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() {
|
||||||
if d == nil || d.State.Final() {
|
if d == nil || d.State.Final() {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -2037,6 +2245,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() {
|
|||||||
h.refuse(err)
|
h.refuse(err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747)
|
||||||
h.moved(d, []Transition{t})
|
h.moved(d, []Transition{t})
|
||||||
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
|
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
|
||||||
// of #221): a merge walked again and then stopped by its group is left out there too.
|
// of #221): a merge walked again and then stopped by its group is left out there too.
|
||||||
@@ -2118,6 +2327,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
|
|||||||
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
|
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// A stop whose change a later walk delivered since, or one its group made for a member delivered since,
|
||||||
|
// holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq
|
||||||
|
// issue 747).
|
||||||
|
if d.coveredStop() || h.stopLapsed(d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
// By the merges it names alone: its own commit may be another's merge it no longer answers.
|
// By the merges it names alone: its own commit may be another's merge it no longer answers.
|
||||||
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
|
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
|
||||||
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
|
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
|
||||||
@@ -2135,6 +2350,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
|
|||||||
return out, why
|
return out, why
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// coverAwaited is whether a stopped merge a walk carries, given as `<repository>@<commit>`, waits for the forge's
|
||||||
|
// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747).
|
||||||
|
// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first
|
||||||
|
// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock.
|
||||||
|
func (h *Holder) coverAwaited(merges []string) (time.Time, bool) {
|
||||||
|
var since time.Time
|
||||||
|
awaited := false
|
||||||
|
for _, d := range h.deliveries {
|
||||||
|
if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, id := range []string{d.ID, d.stoppedFor()} {
|
||||||
|
if id == "" || !h.coverAsked[id] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if at := h.coverAskedSince[id]; !awaited || at.Before(since) {
|
||||||
|
since = at
|
||||||
|
}
|
||||||
|
awaited = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return since, awaited
|
||||||
|
}
|
||||||
|
|
||||||
|
// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped
|
||||||
|
// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so
|
||||||
|
// a forge that cannot answer is loud. Under the lock.
|
||||||
|
func (h *Holder) coverHeld(walk string) string {
|
||||||
|
without, _ := h.carriesStopped(walk)
|
||||||
|
if len(without) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
since, awaited := h.coverAwaited(without)
|
||||||
|
if !awaited {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+
|
||||||
|
"finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)",
|
||||||
|
strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later
|
||||||
|
// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out.
|
||||||
|
const coverAskBound = 10 * time.Minute
|
||||||
|
|
||||||
// lastWhyOf is the reason of a delivery's last transition.
|
// lastWhyOf is the reason of a delivery's last transition.
|
||||||
func lastWhyOf(d *Delivery) string {
|
func lastWhyOf(d *Delivery) string {
|
||||||
if len(d.Transitions) == 0 {
|
if len(d.Transitions) == 0 {
|
||||||
@@ -2170,6 +2430,9 @@ func (h *Holder) leaveStopped() []func() {
|
|||||||
if without, _ := h.carriesStopped(id); len(without) > 0 {
|
if without, _ := h.carriesStopped(id); len(without) > 0 {
|
||||||
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
|
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
|
||||||
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
|
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
|
||||||
|
if hold := h.coverHeld(id); hold != "" {
|
||||||
|
said = "its walk " + id + " is held back: " + hold
|
||||||
|
}
|
||||||
if r := h.leaveRefused[id]; r != "" {
|
if r := h.leaveRefused[id]; r != "" {
|
||||||
said += "; the controller refused: " + r
|
said += "; the controller refused: " + r
|
||||||
}
|
}
|
||||||
@@ -2193,6 +2456,9 @@ func (h *Holder) leaveStopped() []func() {
|
|||||||
delete(h.leaveRefused, id)
|
delete(h.leaveRefused, id)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
if _, awaited := h.coverAwaited(without); awaited {
|
||||||
|
continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747)
|
||||||
|
}
|
||||||
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
|
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
|
||||||
// the ask have gone nowhere.
|
// the ask have gone nowhere.
|
||||||
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
|
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
|
||||||
@@ -2314,18 +2580,28 @@ func againWords(a StopAnswer) string {
|
|||||||
func GroupState(g *Group, members []*Delivery) (string, string) {
|
func GroupState(g *Group, members []*Delivery) (string, string) {
|
||||||
count := map[State]int{}
|
count := map[State]int{}
|
||||||
reached, ended := 0, 0
|
reached, ended := 0, 0
|
||||||
var uncovered []string
|
var uncovered, lapsed []string
|
||||||
|
byID := map[string]*Delivery{}
|
||||||
|
for _, d := range members {
|
||||||
|
byID[d.ID] = d
|
||||||
|
}
|
||||||
for _, d := range members {
|
for _, d := range members {
|
||||||
count[d.State]++
|
|
||||||
if d.State.Final() {
|
if d.State.Final() {
|
||||||
ended++
|
ended++
|
||||||
}
|
}
|
||||||
switch {
|
switch {
|
||||||
case d.State.Reached(), d.State == Superseded && d.CoveredBy != "":
|
case memberReached(d):
|
||||||
|
// Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the
|
||||||
|
// group (novox/hq issue 747).
|
||||||
reached++
|
reached++
|
||||||
|
continue
|
||||||
|
case stopLapsed(d, func(id string) *Delivery { return byID[id] }):
|
||||||
|
lapsed = append(lapsed, d.ID)
|
||||||
|
continue
|
||||||
case d.State == Superseded:
|
case d.State == Superseded:
|
||||||
uncovered = append(uncovered, d.ID)
|
uncovered = append(uncovered, d.ID)
|
||||||
}
|
}
|
||||||
|
count[d.State]++
|
||||||
}
|
}
|
||||||
n := len(members)
|
n := len(members)
|
||||||
switch {
|
switch {
|
||||||
@@ -2335,6 +2611,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
|
|||||||
return "stopped", "a member was stopped"
|
return "stopped", "a member was stopped"
|
||||||
case n > 0 && reached == n:
|
case n > 0 && reached == n:
|
||||||
return "delivered", "every member is delivered"
|
return "delivered", "every member is delivered"
|
||||||
|
case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n:
|
||||||
|
// Stopped for a member delivered since: its own change waits for a later walk of its repository, which then
|
||||||
|
// covers it (novox/hq issue 747).
|
||||||
|
return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+
|
||||||
|
"later walk of its repository", reached, n, strings.Join(lapsed, ", "))
|
||||||
case g.Closed && n > 0 && ended == n:
|
case g.Closed && n > 0 && ended == n:
|
||||||
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
|
return "stopped", supersededWhy + strings.Join(uncovered, ", ")
|
||||||
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
|
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -573,3 +574,49 @@ func TestStalledAndCloseWorkFromTheTable(t *testing.T) {
|
|||||||
t.Fatalf("H2 closed a held delivery: %v", err)
|
t.Fatalf("H2 closed a held delivery: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` also says its pull
|
||||||
|
// request number, full head, base branch and whether it is open; `members` keeps its text form.
|
||||||
|
func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
headA := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||||
|
headB := "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"
|
||||||
|
pb := pr("novox/b", 2, headB, "feat/z")
|
||||||
|
pb.Base = "release/1"
|
||||||
|
w.h.PullUpdated(pr("novox/a", 1, headA, "feat/z"))
|
||||||
|
w.h.PullUpdated(pb)
|
||||||
|
lines := w.h.Groups(false)
|
||||||
|
if len(lines) != 1 || len(lines[0].Details) != 2 || len(lines[0].Members) != 2 {
|
||||||
|
t.Fatalf("groups: %+v", lines)
|
||||||
|
}
|
||||||
|
got := map[string]MemberLine{}
|
||||||
|
for _, m := range lines[0].Details {
|
||||||
|
got[m.Repository] = m
|
||||||
|
}
|
||||||
|
a, b := got["novox/a"], got["novox/b"]
|
||||||
|
if a.Number != 1 || a.Head != headA || a.Base != "main" || !a.Open || a.ID == "" {
|
||||||
|
t.Fatalf("member a: %+v", a)
|
||||||
|
}
|
||||||
|
if b.Number != 2 || b.Head != headB || b.Base != "release/1" || !b.Open {
|
||||||
|
t.Fatalf("member b: %+v", b)
|
||||||
|
}
|
||||||
|
if !strings.Contains(lines[0].Members[0], "(") {
|
||||||
|
t.Fatalf("members lost its text form: %v", lines[0].Members)
|
||||||
|
}
|
||||||
|
// Closed unmerged: still listed while the group is, but not open.
|
||||||
|
w.h.PullClosed(pr("novox/b", 2, headB, "feat/z"))
|
||||||
|
for _, l := range w.h.Groups(true) {
|
||||||
|
for _, m := range l.Details {
|
||||||
|
if m.Repository == "novox/b" && m.Open {
|
||||||
|
t.Fatalf("a closed pull request is open: %+v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// JSON keys are the contract's.
|
||||||
|
raw, _ := json.Marshal(a)
|
||||||
|
for _, k := range []string{`"number":1`, `"head":"` + headA + `"`, `"base":"main"`, `"open":true`} {
|
||||||
|
if !strings.Contains(string(raw), k) {
|
||||||
|
t.Fatalf("%s lacks %s", raw, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
|
||||||
|
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
|
||||||
|
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
|
||||||
|
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
|
||||||
|
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
|
||||||
|
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
|
||||||
|
// still undelivered.
|
||||||
|
//
|
||||||
|
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
|
||||||
|
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
|
||||||
|
|
||||||
|
const (
|
||||||
|
g747 = "fix/a-module-moves-only-when-its-program-changes"
|
||||||
|
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
|
||||||
|
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
|
||||||
|
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
|
||||||
|
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
|
||||||
|
merge307 = "0f76b6e2aaaaaaaa"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
|
||||||
|
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
|
||||||
|
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
|
||||||
|
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
|
||||||
|
|
||||||
|
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
|
||||||
|
func night747With(t *testing.T, contained bool, built []string) *world {
|
||||||
|
t.Helper()
|
||||||
|
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
|
||||||
|
}
|
||||||
|
|
||||||
|
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
|
||||||
|
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
|
||||||
|
t.Helper()
|
||||||
|
at := at747
|
||||||
|
failedAt := at.Add(11 * time.Minute)
|
||||||
|
store := newMemStore()
|
||||||
|
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
|
||||||
|
if built != nil {
|
||||||
|
moved339, unchanged339 = built, map[string]string{}
|
||||||
|
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
|
||||||
|
if !containsString(built, m) {
|
||||||
|
unchanged339[m] = "its build source holds none of the files #339 changed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
|
||||||
|
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
|
||||||
|
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
|
||||||
|
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
|
||||||
|
Unchanged: unchangedRouteProxy747},
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
||||||
|
Why: "its walk failed, carrying its own commit"}}}
|
||||||
|
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
|
||||||
|
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
|
||||||
|
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
|
||||||
|
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
|
||||||
|
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
|
||||||
|
Moved: moved339, Unchanged: unchanged339,
|
||||||
|
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
|
||||||
|
Why: "its walk failed, carrying its own commit"}}}
|
||||||
|
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
|
||||||
|
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
|
||||||
|
for _, d := range []*Delivery{m336, m339, m301} {
|
||||||
|
if err := store.PutDelivery(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
|
||||||
|
OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
|
||||||
|
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
|
||||||
|
if err := store.PutGroup(g); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctlr := newFakeController()
|
||||||
|
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
|
||||||
|
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
|
||||||
|
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
|
||||||
|
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
|
||||||
|
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
|
||||||
|
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
|
||||||
|
forge := newFakeForge()
|
||||||
|
set(forge)
|
||||||
|
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
|
||||||
|
if err := w.h.Reconcile(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func groupState747(w *world) (string, string) {
|
||||||
|
g := w.h.groups[g747]
|
||||||
|
return GroupState(g, w.h.membersOf(g))
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
|
||||||
|
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
|
||||||
|
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
|
||||||
|
// own walk was resumed to done, is covered by it too.
|
||||||
|
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
|
||||||
|
w := night747(t, true)
|
||||||
|
d := w.delivery(id336)
|
||||||
|
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
|
||||||
|
!strings.Contains(desc, "plan-339") {
|
||||||
|
t.Fatalf("#336's status reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
|
||||||
|
t.Fatalf("the forge was told %q of #336", s)
|
||||||
|
}
|
||||||
|
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
|
||||||
|
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state == "failed" || state == "stopped" {
|
||||||
|
t.Fatalf("the group reads %s: %s", state, why)
|
||||||
|
}
|
||||||
|
// Told once, and kept: a restart tells nothing again.
|
||||||
|
notes := len(w.forge.notesOn(merge336))
|
||||||
|
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
|
||||||
|
again.settleAll()
|
||||||
|
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
|
||||||
|
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
|
||||||
|
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
|
||||||
|
// group waits for that walk to deliver the catalogue member's change.
|
||||||
|
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
|
||||||
|
w := night747(t, true)
|
||||||
|
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
|
||||||
|
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
|
||||||
|
t.Fatalf("the catalogue's walk is held back for %v", stopped)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
|
||||||
|
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
|
||||||
|
}
|
||||||
|
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
|
||||||
|
// delivered.
|
||||||
|
w.forge.contains[merge301+" in "+merge307] = true
|
||||||
|
r := w.ctl.walk("plan-cat")
|
||||||
|
sent := w.now
|
||||||
|
r.Modules = map[string]*WalkModule{}
|
||||||
|
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
|
||||||
|
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
||||||
|
}
|
||||||
|
r.State = walkDone
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
w.later(time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
|
||||||
|
t.Fatalf("the catalogue member reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
if state, why := groupState747(w); state != "delivered" {
|
||||||
|
t.Fatalf("the group reads %s: %s", state, why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
|
||||||
|
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
|
||||||
|
// the stopped member's merge is asked to leave it out.
|
||||||
|
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
|
||||||
|
w := night747(t, false)
|
||||||
|
d := w.delivery(id336)
|
||||||
|
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
|
||||||
|
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
|
||||||
|
}
|
||||||
|
if state, _ := groupState747(w); state != "failed" {
|
||||||
|
t.Fatalf("the group reads %s", state)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
||||||
|
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
|
||||||
|
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
|
||||||
|
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
|
||||||
|
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
|
||||||
|
w := night747With(t, true, []string{"mesh-controller"})
|
||||||
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
||||||
|
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
|
||||||
|
}
|
||||||
|
if state, _ := groupState747(w); state != "failed" {
|
||||||
|
t.Fatalf("the group reads %s", state)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 {
|
||||||
|
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
|
||||||
|
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
|
||||||
|
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
|
||||||
|
// is asked to leave the stopped merge out, as with no later walk at all.
|
||||||
|
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
set func(*fakeForge)
|
||||||
|
}{
|
||||||
|
{"erroring", func(f *fakeForge) { f.containsDown = true }},
|
||||||
|
{"not answering for the pair", func(*fakeForge) {}},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
w := night747Over(t, c.set, nil)
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
|
||||||
|
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
|
||||||
|
}
|
||||||
|
said := ""
|
||||||
|
for _, l := range w.h.WalksWaiting() {
|
||||||
|
if l.Walk == "plan-cat" {
|
||||||
|
said += l.Says
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
|
||||||
|
w.settleAll()
|
||||||
|
for _, l := range w.h.Stalled() {
|
||||||
|
if l.Walk == "plan-cat" {
|
||||||
|
said += " | " + l.Says
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Count(said, "the forge is asked since") != 2 {
|
||||||
|
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
w.later(coverAskBound + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
|
||||||
|
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,292 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
|
||||||
|
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
|
||||||
|
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
|
||||||
|
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
|
||||||
|
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
|
||||||
|
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
|
||||||
|
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
|
||||||
|
//
|
||||||
|
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
|
||||||
|
|
||||||
|
const (
|
||||||
|
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
|
||||||
|
plan753 = "plan-1792012015539062661"
|
||||||
|
)
|
||||||
|
|
||||||
|
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
|
||||||
|
|
||||||
|
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
|
||||||
|
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
|
||||||
|
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
|
||||||
|
|
||||||
|
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
|
||||||
|
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
|
||||||
|
t.Helper()
|
||||||
|
return night753Over(t, stop, nil, nil, more...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
|
||||||
|
// tick.
|
||||||
|
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
|
||||||
|
t.Helper()
|
||||||
|
at := at747
|
||||||
|
store := newMemStore()
|
||||||
|
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
|
||||||
|
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
|
||||||
|
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
|
||||||
|
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
|
||||||
|
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
|
||||||
|
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
|
||||||
|
if stop.By == byOwner {
|
||||||
|
m301.StoppedFor = id336
|
||||||
|
}
|
||||||
|
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
|
||||||
|
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
|
||||||
|
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
|
||||||
|
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
|
||||||
|
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
|
||||||
|
CoverTold: true,
|
||||||
|
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
|
||||||
|
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
|
||||||
|
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
|
||||||
|
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
|
||||||
|
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
|
||||||
|
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
|
||||||
|
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
|
||||||
|
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
|
||||||
|
if err := store.PutDelivery(d); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
inGroup, order := []*Delivery{m301, m336}, []string{id336}
|
||||||
|
for _, d := range more {
|
||||||
|
if d.Group == g747 {
|
||||||
|
inGroup, order = append(inGroup, d), append(order, d.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
order = append(order, id301)
|
||||||
|
var ids []string
|
||||||
|
for _, d := range inGroup {
|
||||||
|
ids = append(ids, d.ID)
|
||||||
|
}
|
||||||
|
g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering",
|
||||||
|
OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
|
||||||
|
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
|
||||||
|
if err := store.PutGroup(g); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctlr := newFakeController()
|
||||||
|
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
|
||||||
|
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
|
||||||
|
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
|
||||||
|
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
|
||||||
|
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
|
||||||
|
{Repository: cat747, Commit: merge307}}}})
|
||||||
|
if len(merges) > 0 {
|
||||||
|
r := ctlr.walk(plan753)
|
||||||
|
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
|
||||||
|
ctlr.put(r)
|
||||||
|
}
|
||||||
|
if set != nil {
|
||||||
|
set(ctlr)
|
||||||
|
}
|
||||||
|
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
|
||||||
|
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
|
||||||
|
if err := w.h.Reconcile(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
|
||||||
|
func groupStop753() Transition {
|
||||||
|
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
|
||||||
|
}
|
||||||
|
|
||||||
|
func saidOf(lines []StalledLine, walk string) (string, State) {
|
||||||
|
said, state := "", State("")
|
||||||
|
for _, l := range lines {
|
||||||
|
if l.Walk == walk {
|
||||||
|
said, state = said+l.Says, l.State
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return said, state
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
|
||||||
|
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
|
||||||
|
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
|
||||||
|
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
|
||||||
|
w := night753(t, groupStop753())
|
||||||
|
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
|
||||||
|
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
|
||||||
|
}
|
||||||
|
d := w.delivery(id301)
|
||||||
|
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
|
||||||
|
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
|
||||||
|
}
|
||||||
|
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
|
||||||
|
t.Fatalf("the walk reads unheld: %q", said)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
|
||||||
|
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
|
||||||
|
}
|
||||||
|
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
|
||||||
|
w.forge.contains[merge301+" in "+merge307] = true
|
||||||
|
r := w.ctl.walk(plan753)
|
||||||
|
sent := w.now
|
||||||
|
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
|
||||||
|
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
|
||||||
|
}
|
||||||
|
r.State = walkDone
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
w.later(time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
|
||||||
|
t.Fatalf("#301 reads %s %q", state, desc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
|
||||||
|
// the walk's, never a way past a live delivery it carries.
|
||||||
|
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
|
||||||
|
at := at747
|
||||||
|
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
|
||||||
|
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
|
||||||
|
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
|
||||||
|
Waits: true, Created: at.Add(46 * time.Minute)}}
|
||||||
|
w := night753With(t, groupStop753(), held)
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
|
||||||
|
// never let go for it.
|
||||||
|
func TestAPersonsStopGivesNoWord(t *testing.T) {
|
||||||
|
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
|
||||||
|
Why: "stopped by hand"})
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
|
||||||
|
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
|
||||||
|
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
|
||||||
|
// only that nothing holds it.
|
||||||
|
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
|
||||||
|
w := night753(t, groupStop753())
|
||||||
|
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
|
||||||
|
// gives it its word, and no live one holds it.
|
||||||
|
r := w.ctl.walk(plan753)
|
||||||
|
r.Delivery.Go = nil
|
||||||
|
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
|
||||||
|
w.h.WalkMoved(w.ctl.put(r))
|
||||||
|
said, state := saidOf(w.h.WalksWaiting(), plan753)
|
||||||
|
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
|
||||||
|
}
|
||||||
|
w.settleAll()
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, _ = saidOf(w.h.Stalled(), plan753)
|
||||||
|
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
|
||||||
|
t.Fatalf("`stalled` reads %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
|
||||||
|
func m299() *Delivery {
|
||||||
|
at := at747
|
||||||
|
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
|
||||||
|
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
|
||||||
|
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
|
||||||
|
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
|
||||||
|
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
|
||||||
|
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and
|
||||||
|
// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land
|
||||||
|
// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and
|
||||||
|
// is let go.
|
||||||
|
func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) {
|
||||||
|
w := night753With(t, groupStop753(), m299())
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
said, state := saidOf(w.h.WalksWaiting(), plan753)
|
||||||
|
if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") {
|
||||||
|
t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said)
|
||||||
|
}
|
||||||
|
w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil,
|
||||||
|
m299())
|
||||||
|
if !containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
|
||||||
|
// the grace `stalled` lists the walk, with the refusal.
|
||||||
|
func TestARefusedWordIsListedByStalled(t *testing.T) {
|
||||||
|
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, state := saidOf(w.h.Stalled(), plan753)
|
||||||
|
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
|
||||||
|
t.Fatalf("`stalled` reads %s %q", state, said)
|
||||||
|
}
|
||||||
|
// Once the controller takes it, the walk is let go.
|
||||||
|
w.ctl.refuseDeliver = false
|
||||||
|
w.settleAll()
|
||||||
|
if !containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
|
||||||
|
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
|
||||||
|
// grace `stalled` lists the walk, saying why.
|
||||||
|
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
|
||||||
|
at := at747
|
||||||
|
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
|
||||||
|
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
|
||||||
|
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
|
||||||
|
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
|
||||||
|
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
|
||||||
|
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
|
||||||
|
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
|
||||||
|
if containsString(w.ctl.delivers, plan753) {
|
||||||
|
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
w.later(matchGrace + time.Minute)
|
||||||
|
w.settleAll()
|
||||||
|
said, _ := saidOf(w.h.Stalled(), plan753)
|
||||||
|
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
|
||||||
|
!strings.Contains(said, "whose delivery was stopped") {
|
||||||
|
t.Fatalf("`stalled` reads %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was
|
||||||
|
// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been
|
||||||
|
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped
|
||||||
|
// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery
|
||||||
|
// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and
|
||||||
|
// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it.
|
||||||
|
//
|
||||||
|
// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its
|
||||||
|
// commit's delivery for another pull request with the same head: it is started over, under the same id, for the
|
||||||
|
// pull request that announces or merges it next.
|
||||||
|
|
||||||
|
const (
|
||||||
|
head783, merge783 = "d305d305d305d305", "d315d315d315d315"
|
||||||
|
head783b, merge783b = "d306d306d306d306", "d316d316d316d316"
|
||||||
|
head783c = "d307d307d307d307"
|
||||||
|
head783d, merge783d = "d308d308d308d308", "d318d318d318d318"
|
||||||
|
catalog783 = "novox/mesh-catalog"
|
||||||
|
branch783 = "adr-359-mesh-mail-733"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
|
||||||
|
// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it,
|
||||||
|
// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop.
|
||||||
|
func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 305, head783, branch783))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go"))
|
||||||
|
if d := w.delivery(id); d.State == Stopped || d.Number != 315 {
|
||||||
|
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number)
|
||||||
|
}
|
||||||
|
if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") {
|
||||||
|
t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.Checked(verdict(catalog783, 315, head783, "pass"))
|
||||||
|
if d := w.delivery(id); d.State != Ready {
|
||||||
|
t.Fatalf("a passing verdict for #315 left it %s", d.State)
|
||||||
|
}
|
||||||
|
|
||||||
|
walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true))
|
||||||
|
w.h.WalkMoved(walk)
|
||||||
|
w.h.PullMerged(merged(catalog783, 315, head783, merge783))
|
||||||
|
if d := w.delivery(id); d.State != Published || d.Number != 315 {
|
||||||
|
t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" {
|
||||||
|
t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
if d := w.delivery(id); d.State != Delivering {
|
||||||
|
t.Fatalf("let go, it is %s; want delivering", d.State)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never
|
||||||
|
// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in
|
||||||
|
// PullUpdated, so a merge heard without the open announcement is not lost either.
|
||||||
|
func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783b)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true))
|
||||||
|
w.h.WalkMoved(walk)
|
||||||
|
m := merged(catalog783, 316, head783b, merge783b)
|
||||||
|
m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"}
|
||||||
|
w.h.PullMerged(m)
|
||||||
|
if d := w.delivery(id); d.State == Stopped || d.Number != 316 {
|
||||||
|
t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316",
|
||||||
|
d.State, d.Number)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.settleAll()
|
||||||
|
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" {
|
||||||
|
t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped
|
||||||
|
// delivery ... is untouched": the same pull request announced again after its own close is the ordinary
|
||||||
|
// 'announced again' path, never a start-over, and stays stopped.
|
||||||
|
func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783c)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
|
||||||
|
if d := w.delivery(id); d.State != Stopped || d.Number != 307 {
|
||||||
|
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's
|
||||||
|
// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above)
|
||||||
|
// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it
|
||||||
|
// happened still falls under the history rule (novox/hq issue 313) and is never held for a person.
|
||||||
|
func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
id := IDOf(catalog783, head783d)
|
||||||
|
|
||||||
|
w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go"))
|
||||||
|
w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d"))
|
||||||
|
if w.state(id) != Stopped {
|
||||||
|
t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
m := merged(catalog783, 318, head783d, merge783d)
|
||||||
|
m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339)
|
||||||
|
w.h.PullMerged(m)
|
||||||
|
|
||||||
|
d := w.delivery(id)
|
||||||
|
if d.State == Stopped || d.Number != 318 {
|
||||||
|
t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+
|
||||||
|
"want started over for #318", d.State, d.Number)
|
||||||
|
}
|
||||||
|
if !d.FromForge {
|
||||||
|
t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false")
|
||||||
|
}
|
||||||
|
if d.History == "" {
|
||||||
|
t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute)
|
||||||
|
}
|
||||||
|
if d.State == Held {
|
||||||
|
t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -83,6 +83,14 @@ type Delivery struct {
|
|||||||
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
|
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
|
||||||
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
|
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
|
||||||
CoverTold bool `json:"cover_told,omitempty"`
|
CoverTold bool `json:"cover_told,omitempty"`
|
||||||
|
// DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose
|
||||||
|
// commit contains its merge and which built every module it moves — each it did not build, its own walk said
|
||||||
|
// unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered
|
||||||
|
// stop's does (issue 480), and it reads covered: its group counts it as delivered.
|
||||||
|
DeliveredBy string `json:"delivered_by,omitempty"`
|
||||||
|
// StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that
|
||||||
|
// member's change reached the machines, the stop has lapsed, and holds no walk of its repository back.
|
||||||
|
StoppedFor string `json:"stopped_for,omitempty"`
|
||||||
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
|
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
|
||||||
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
|
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
|
||||||
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
|
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
|
||||||
@@ -303,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool {
|
|||||||
|
|
||||||
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
|
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
|
||||||
// pull request the forge may no longer hold — or holds under its number another one.
|
// pull request the forge may no longer hold — or holds under its number another one.
|
||||||
// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules
|
// coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop
|
||||||
// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every
|
// because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a
|
||||||
// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped —
|
// later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running
|
||||||
// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq
|
// it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk
|
||||||
// issue 480). A stop for any other reason never reads covered.
|
// of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state
|
||||||
|
// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or
|
||||||
|
// walk. A person's stop with a cover found only while it was held keeps its error.
|
||||||
func (d *Delivery) coveredStop() bool {
|
func (d *Delivery) coveredStop() bool {
|
||||||
return d.State == Stopped && d.Stranded != "" && d.CoveredBy != ""
|
switch d.State {
|
||||||
|
case Stopped:
|
||||||
|
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
|
||||||
|
case Failed:
|
||||||
|
return d.DeliveredBy != ""
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`.
|
// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
|
||||||
func (d *Delivery) coveredStopSaid() string {
|
func (d *Delivery) coveredStopSaid() string {
|
||||||
return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy
|
ended := "its stop"
|
||||||
|
if d.State == Failed {
|
||||||
|
ended = "its failure"
|
||||||
|
}
|
||||||
|
if d.Stranded != "" && d.CoveredBy != "" {
|
||||||
|
return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy
|
||||||
|
}
|
||||||
|
return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy
|
||||||
|
}
|
||||||
|
|
||||||
|
// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to
|
||||||
|
// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442),
|
||||||
|
// or a stop or failure covered since (issues 480 and 747).
|
||||||
|
func memberReached(d *Delivery) bool {
|
||||||
|
return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the
|
||||||
|
// stop's reason for a stop made before ("its group <id> stopped: <member> …").
|
||||||
|
func (d *Delivery) stoppedFor() string {
|
||||||
|
if d.StoppedFor != "" || d.State != Stopped || d.Group == "" {
|
||||||
|
return d.StoppedFor
|
||||||
|
}
|
||||||
|
prefix := "its group " + d.Group + " stopped: "
|
||||||
|
for i := len(d.Transitions) - 1; i >= 0; i-- {
|
||||||
|
t := d.Transitions[i]
|
||||||
|
if t.Event != EvStop {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ")
|
||||||
|
return strings.TrimRight(member, ",;:")
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's
|
||||||
|
// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own
|
||||||
|
// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find
|
||||||
|
// gives a delivery by its id.
|
||||||
|
func stopLapsed(d *Delivery, find func(string) *Delivery) bool {
|
||||||
|
if d.State != Stopped || d.coveredStop() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
by := d.stoppedFor()
|
||||||
|
if by == "" || by == d.ID {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
o := find(by)
|
||||||
|
return o != nil && memberReached(o)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d *Delivery) shownOnItsPullRequest() bool {
|
func (d *Delivery) shownOnItsPullRequest() bool {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// A recheck puts the merge check's statuses back to pending before it asks the check: the forge keeps the
|
// A recheck puts the merge check's statuses back to pending before it asks the check: the forge keeps the
|
||||||
@@ -95,3 +96,80 @@ func TestARecheckTheForgeCannotTakeIsRefused(t *testing.T) {
|
|||||||
t.Fatalf("refused, it moved to %s", s)
|
t.Fatalf("refused, it moved to %s", s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 867, ADR 0323 §6: "the composed check runs again when a member is rechecked". A group whose composed
|
||||||
|
// check failed stayed rejected after a member was rechecked and passed, because the kept verdict was for the same heads
|
||||||
|
// and nothing marked it out of date.
|
||||||
|
func TestAMembersRecheckAsksTheGroupsComposedCheckAgain(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z"))
|
||||||
|
w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z"))
|
||||||
|
w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass"))
|
||||||
|
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
|
||||||
|
w.settleAll()
|
||||||
|
groupAsks := func() (n int) {
|
||||||
|
for _, c := range w.ctl.checks {
|
||||||
|
if strings.HasPrefix(c, "feat/z:") {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if groupAsks() != 1 {
|
||||||
|
t.Fatalf("the group was asked %v", w.ctl.checks)
|
||||||
|
}
|
||||||
|
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"})
|
||||||
|
w.settleAll()
|
||||||
|
if state, _ := GroupState(w.h.groups["feat/z"], w.h.membersOf(w.h.groups["feat/z"])); state != "rejected" {
|
||||||
|
t.Fatalf("the group is %s", state)
|
||||||
|
}
|
||||||
|
if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The member's own check passes again, at the same head, a while later.
|
||||||
|
w.later(5 * time.Minute)
|
||||||
|
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
|
||||||
|
w.settleAll()
|
||||||
|
if groupAsks() != 2 {
|
||||||
|
t.Fatalf("a member's recheck did not ask the group again: %v", w.ctl.checks)
|
||||||
|
}
|
||||||
|
if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.Asked == "check-1" {
|
||||||
|
t.Fatalf("the old verdict still stands: %+v", g.Check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A recheck made while the group's composed check is still running is not lost: the old run's answer lands, and the
|
||||||
|
// group is still asked again once the member is ready.
|
||||||
|
func TestARecheckWhileTheComposedCheckRunsAsksTheGroupAgain(t *testing.T) {
|
||||||
|
w := newWorld(t)
|
||||||
|
w.h.PullUpdated(pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z"))
|
||||||
|
w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z"))
|
||||||
|
w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass"))
|
||||||
|
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
|
||||||
|
w.settleAll() // the composed check check-1 is asked and running: no verdict yet
|
||||||
|
if _, err := w.h.Recheck(IDOf("novox/b", "b2b2b2b2b2b2b2"), "the judge was fixed", "a person"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The old run answers.
|
||||||
|
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "they do not compose"})
|
||||||
|
w.later(5 * time.Minute)
|
||||||
|
w.h.Checked(verdict("novox/b", 2, "b2b2b2b2b2b2b2", "pass"))
|
||||||
|
w.settleAll()
|
||||||
|
n := 0
|
||||||
|
for _, c := range w.ctl.checks {
|
||||||
|
if strings.HasPrefix(c, "feat/z:") {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n != 2 {
|
||||||
|
t.Fatalf("the group was not asked again after a recheck made while its check ran: %v", w.ctl.checks)
|
||||||
|
}
|
||||||
|
if g := w.h.groups["feat/z"]; g.Check == nil || g.Check.Verdict != "" || g.Check.OutOfDate != "" {
|
||||||
|
t.Fatalf("the old run's answer stands: %+v", g.Check)
|
||||||
|
}
|
||||||
|
// The old run answering again is history.
|
||||||
|
w.h.Checked(CheckedEvent{Group: "feat/z", ID: "check-1", Verdict: "fail", Summary: "late"})
|
||||||
|
if g := w.h.groups["feat/z"]; g.Check.Verdict != "" {
|
||||||
|
t.Fatalf("a late answer of the old run was taken: %+v", g.Check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string {
|
|||||||
return "its walk: " + clip(d.Walk.Note, 120)
|
return "its walk: " + clip(d.Walk.Note, 120)
|
||||||
}
|
}
|
||||||
case Stopped:
|
case Stopped:
|
||||||
|
if h.stopLapsed(d) {
|
||||||
|
return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() +
|
||||||
|
", whose change was delivered since (novox/hq issue 747)"
|
||||||
|
}
|
||||||
if strandedUncovered(d) {
|
if strandedUncovered(d) {
|
||||||
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
|
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
|
||||||
"listed by stalled until one does (novox/hq issue 480)"
|
"listed by stalled until one does (novox/hq issue 480)"
|
||||||
@@ -150,6 +154,21 @@ type GroupLine struct {
|
|||||||
Contradictions []Contradiction `json:"contradictions,omitempty"`
|
Contradictions []Contradiction `json:"contradictions,omitempty"`
|
||||||
Check string `json:"composed,omitempty"`
|
Check string `json:"composed,omitempty"`
|
||||||
Members []string `json:"members"`
|
Members []string `json:"members"`
|
||||||
|
// Details are the members again, one object each, with what a reader needs to find the pull request's
|
||||||
|
// siblings: its number, full head, base branch and whether it is open (novox/hq ADR 0382 §1). Additive:
|
||||||
|
// `members` keeps its text form for older readers.
|
||||||
|
Details []MemberLine `json:"member_details,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemberLine is one member of a group, as `groups` lists it beside its text line.
|
||||||
|
type MemberLine struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
State State `json:"state"`
|
||||||
|
Number int `json:"number"`
|
||||||
|
Head string `json:"head"`
|
||||||
|
Base string `json:"base"`
|
||||||
|
Open bool `json:"open"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Groups is the `groups` verb.
|
// Groups is the `groups` verb.
|
||||||
@@ -170,6 +189,9 @@ func (h *Holder) Groups(all bool) []GroupLine {
|
|||||||
Contradictions: g.Contradictions}
|
Contradictions: g.Contradictions}
|
||||||
for _, d := range members {
|
for _, d := range members {
|
||||||
l.Members = append(l.Members, fmt.Sprintf("%s (%s)", d.ID, d.State))
|
l.Members = append(l.Members, fmt.Sprintf("%s (%s)", d.ID, d.State))
|
||||||
|
l.Details = append(l.Details, MemberLine{ID: d.ID, Repository: d.Repository, State: d.State,
|
||||||
|
Number: d.Number, Head: d.Commit, Base: d.Base,
|
||||||
|
Open: !d.State.Final() && d.MergedAs == "" && !d.ClosedUnmerged})
|
||||||
}
|
}
|
||||||
if g.Check != nil {
|
if g.Check != nil {
|
||||||
l.Check = g.Check.Verdict
|
l.Check = g.Check.Verdict
|
||||||
@@ -288,6 +310,16 @@ func (h *Holder) Recheck(id, why, by string) (string, error) {
|
|||||||
return "", fmt.Errorf("%v (its merge check says pending; the forge's next announcement asks it)", err)
|
return "", fmt.Errorf("%v (its merge check says pending; the forge's next announcement asks it)", err)
|
||||||
}
|
}
|
||||||
d.Check = nil
|
d.Check = nil
|
||||||
|
// The composed check of its group goes out of date (novox/hq ADR 0323 §6, issue 867): it was judged for these very
|
||||||
|
// heads (running or answered: a run's answer is taken by its id and must not stand for the recheck), so without this the kept verdict (a failure above all) stands and the group is never asked again. It is
|
||||||
|
// asked again once the member is judged again, when every member is ready.
|
||||||
|
if g := h.groups[d.Group]; g != nil && g.Check != nil {
|
||||||
|
now := h.Now()
|
||||||
|
g.Check.OutOfDate = fmt.Sprintf("composed check out of date: %s was rechecked (%s)", d.ID, why)
|
||||||
|
g.Check.OutOfDateAt = now
|
||||||
|
g.Updated = now
|
||||||
|
h.keepGroup(g)
|
||||||
|
}
|
||||||
h.moved(d, []Transition{t})
|
h.moved(d, []Transition{t})
|
||||||
m := MemberOf(d)
|
m := MemberOf(d)
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ operator, it is accepted only on the operator's Approve, given on a channel that
|
|||||||
| `links` | `number`, `kind` | the record's typed links, out and in |
|
| `links` | `number`, `kind` | the record's typed links, out and in |
|
||||||
| `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line |
|
| `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line |
|
||||||
| `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted |
|
| `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted |
|
||||||
| `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`). Each version proposed is then reviewed by the knowledge base itself (below) |
|
| `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`), but an active research effort takes new versions freely, each reviewed as a proposal. Each version proposed is then reviewed by the knowledge base itself (below) |
|
||||||
| `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number |
|
| `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number |
|
||||||
| `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number |
|
| `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number |
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// effortAsk is a research effort proposal with the given documents.
|
||||||
|
func effortAsk(n int, status string, docs ...Section) ProposeAsk {
|
||||||
|
return ProposeAsk{Kind: KindResearch, Number: n, Title: "A vendor-neutral inference layer", Date: "2026-10-11",
|
||||||
|
Deciders: []string{"jochen"}, Status: status, Sections: docs,
|
||||||
|
Links: []LinkAsk{{Kind: LinkTouches, To: "0290"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 844: an active research effort takes new versions freely, each reviewed as a proposal.
|
||||||
|
func TestAnActiveEffortTakesANewVersionWithANewDocument(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
|
||||||
|
key := Key{KindResearch, r.Number}
|
||||||
|
reviews := len(w.judge.reviews)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
next.Why = "findings added"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatalf("an active effort refused a new version: %v", err)
|
||||||
|
}
|
||||||
|
e := mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending == nil || e.Current.Section("01-findings.md") != "" {
|
||||||
|
t.Fatalf("the new version was served before its review")
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
q := w.judge.reviews[reviews]
|
||||||
|
if strings.Contains(q, "a correction of fact (a progressive insight)") {
|
||||||
|
t.Fatalf("the review judged a new version as an insight:\n%.400s", q)
|
||||||
|
}
|
||||||
|
e = mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending != nil || e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "A new finding." ||
|
||||||
|
len(e.Current.Insights) != 0 {
|
||||||
|
t.Fatalf("after the review: %s, pending %v, insights %v", e.Current.Status, e.Pending != nil, e.Current.Insights)
|
||||||
|
}
|
||||||
|
h, err := w.k.History(key, 0, 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, v := range h.Versions {
|
||||||
|
if v.Insight {
|
||||||
|
t.Errorf("version %d is marked an insight", v.N)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAGraduatedEffortAndADecisionStillTakeOnlyAnInsight(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
grad := effortAsk(r.Number, StatusGraduated, overview)
|
||||||
|
grad.Links = append(grad.Links, LinkAsk{Kind: LinkBecame, To: "0290"})
|
||||||
|
w.accepted(t, "author@g14", grad)
|
||||||
|
next := effortAsk(r.Number, StatusGraduated, overview, Section{Name: "01-findings.md", Text: "More."})
|
||||||
|
next.Links = grad.Links
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "progressive insight") {
|
||||||
|
t.Fatalf("a graduated effort took a plain version: %v", err)
|
||||||
|
}
|
||||||
|
n := w.reserved(t, "author@g14")
|
||||||
|
w.accepted(t, "author@g14", valid(n))
|
||||||
|
p := valid(n)
|
||||||
|
p.Sections[2].Text = "Changed."
|
||||||
|
if _, err := w.k.Propose("author@g14", p); err == nil || !strings.Contains(err.Error(), "progressive insight") {
|
||||||
|
t.Fatalf("a decision took a plain version: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An old active effort's new text is held to the rules as of the day it is proposed, not the day the effort began.
|
||||||
|
func TestANewVersionOfAnOldActiveEffortIsHeldToTodaysRules(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
first := effortAsk(r.Number, StatusActive, overview)
|
||||||
|
first.Date = "2026-10-01"
|
||||||
|
w.accepted(t, "author@g14", first)
|
||||||
|
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "The control plane wrote them."})
|
||||||
|
next.Date = "2026-10-01"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "retired-word") {
|
||||||
|
t.Fatalf("a retired word in new text of an old effort: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAChangesVerdictLeavesTheServedVersionServedAndGraduatingNeedsABecameLink(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
|
||||||
|
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
|
||||||
|
key := Key{KindResearch, r.Number}
|
||||||
|
w.judge.review = `{"verdict":"changes","findings":["01-findings.md: say where the finding comes from"]}`
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
e := mustGet(t, w.k.Store, key)
|
||||||
|
if e.Pending == nil || e.Pending.Reviews[len(e.Pending.Reviews)-1].Verdict != VerdictChanges ||
|
||||||
|
e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "" {
|
||||||
|
t.Fatalf("after changes: pending %v, current %+v", e.Pending != nil, e.Current.Sections)
|
||||||
|
}
|
||||||
|
grad := effortAsk(r.Number, StatusGraduated, overview)
|
||||||
|
if _, err := w.k.Propose("author@g14", grad); err == nil || !strings.Contains(err.Error(), "became") {
|
||||||
|
t.Fatalf("graduated without a became link: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An earlier marked insight in an old effort's text does not predate the effort, and a new version is accepted.
|
||||||
|
func TestAnOldActiveEffortWithAMarkedInsightTakesANewVersion(t *testing.T) {
|
||||||
|
w := imported(t, newMemStore())
|
||||||
|
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
|
||||||
|
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be.\n\n**Progressive insight — 2026-10-05.** A fact was corrected."}
|
||||||
|
first := effortAsk(r.Number, StatusActive, overview)
|
||||||
|
first.Date = "2026-10-01"
|
||||||
|
w.accepted(t, "author@g14", first)
|
||||||
|
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
|
||||||
|
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
|
||||||
|
next.Date = "2026-10-01"
|
||||||
|
if _, err := w.k.Propose("author@g14", next); err != nil {
|
||||||
|
t.Fatalf("a new version with an earlier marked insight: %v", err)
|
||||||
|
}
|
||||||
|
w.reviewAll(t)
|
||||||
|
e := mustGet(t, w.k.Store, Key{KindResearch, r.Number})
|
||||||
|
if e.Pending != nil || e.Current.Section("01-findings.md") != "A new finding." || e.Current.Date != "2026-10-01" {
|
||||||
|
t.Fatalf("after the review: pending %v, date %s", e.Pending != nil, e.Current.Date)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -357,7 +357,7 @@ func (k *KB) Get(a GetAsk) (*Got, error) {
|
|||||||
g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
|
g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
|
||||||
}
|
}
|
||||||
if e.Pending != nil && a.Version == 0 {
|
if e.Pending != nil && a.Version == 0 {
|
||||||
g.Pending = fmt.Sprintf("version %d, a progressive insight, waits for its review: history shows what it changes", e.PendingVersion)
|
g.Pending = fmt.Sprintf("version %d, a progressive insight or a new version of an active effort, waits for its review: history shows what it changes", e.PendingVersion)
|
||||||
}
|
}
|
||||||
if r.Ask != nil {
|
if r.Ask != nil {
|
||||||
g.Waiting = "the operator's ask: " + r.Ask.Said
|
g.Waiting = "the operator's ask: " + r.Ask.Said
|
||||||
|
|||||||
@@ -335,7 +335,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+
|
return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+
|
||||||
"runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller)
|
"runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(a.Insight) == "" {
|
// An active research effort is still being researched: it takes new versions freely, each reviewed as a
|
||||||
|
// proposal while the served version stays (novox/hq issue 844). Every other accepted record needs an insight.
|
||||||
|
revision := a.Kind == KindResearch && held.Current.Status == StatusActive && strings.TrimSpace(a.Insight) == ""
|
||||||
|
if strings.TrimSpace(a.Insight) == "" && !revision {
|
||||||
return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
|
return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
|
||||||
"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
|
"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
|
||||||
}
|
}
|
||||||
@@ -349,8 +352,14 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}}
|
r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}}
|
||||||
r.Date = held.Current.Date
|
r.Date = held.Current.Date
|
||||||
r.Effort = a.Status
|
r.Effort = a.Status
|
||||||
|
if revision {
|
||||||
|
r.Pending = &PendingState{}
|
||||||
|
}
|
||||||
if why == "" {
|
if why == "" {
|
||||||
why = "a progressive insight proposed"
|
why = "a progressive insight proposed"
|
||||||
|
if revision {
|
||||||
|
why = "a new version proposed"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status)
|
return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status)
|
||||||
@@ -370,7 +379,13 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
if !current && a.Kind == KindResearch && r.Effort == "" {
|
if !current && a.Kind == KindResearch && r.Effort == "" {
|
||||||
r.Effort = held.Current.Status
|
r.Effort = held.Current.Status
|
||||||
}
|
}
|
||||||
refusals = append(refusals, JudgeRecord(r, held, k.lookup(ctx), ref)...)
|
// A new version of an active effort is new text: the dated rules hold it from the day it is proposed, while the
|
||||||
|
// record keeps its own date (novox/hq issue 844).
|
||||||
|
day := r.Date
|
||||||
|
if r.Pending != nil && !r.IsInsight() {
|
||||||
|
day = k.now().Format("2006-01-02")
|
||||||
|
}
|
||||||
|
refusals = append(refusals, JudgeRecordOn(r, held, k.lookup(ctx), ref, day)...)
|
||||||
if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes {
|
if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes {
|
||||||
refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)})
|
refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)})
|
||||||
}
|
}
|
||||||
@@ -381,7 +396,7 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
}
|
}
|
||||||
return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- "))
|
return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- "))
|
||||||
}
|
}
|
||||||
v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: !current}
|
v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: r.IsInsight()}
|
||||||
if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil {
|
if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil {
|
||||||
return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err)
|
return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err)
|
||||||
}
|
}
|
||||||
@@ -397,7 +412,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
|
|||||||
if r.NeedsOperator {
|
if r.NeedsOperator {
|
||||||
says += ", and then the operator's Approve"
|
says += ", and then the operator's Approve"
|
||||||
}
|
}
|
||||||
if !current {
|
if !current && !r.IsInsight() {
|
||||||
|
says = fmt.Sprintf("A new version of %s is proposed as version %d. The knowledge base reviews it now; the served version stays "+
|
||||||
|
"until that review approves it", key, n)
|
||||||
|
} else if !current {
|
||||||
says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+
|
says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+
|
||||||
"version is served until that review approves it", key, n)
|
"version is served until that review approves it", key, n)
|
||||||
}
|
}
|
||||||
@@ -416,8 +434,11 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
|
|||||||
at := k.now()
|
at := k.now()
|
||||||
if insight {
|
if insight {
|
||||||
in := r.Pending.Insight
|
in := r.Pending.Insight
|
||||||
|
isInsight := r.IsInsight()
|
||||||
r.Pending = nil
|
r.Pending = nil
|
||||||
r.Insights = append(r.Insights, in)
|
if isInsight {
|
||||||
|
r.Insights = append(r.Insights, in)
|
||||||
|
}
|
||||||
r.Status = held.Current.Status
|
r.Status = held.Current.Status
|
||||||
if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded {
|
if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded {
|
||||||
r.Status = r.Effort
|
r.Status = r.Effort
|
||||||
@@ -442,7 +463,7 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
|
|||||||
if warrant != "" {
|
if warrant != "" {
|
||||||
why += "; " + warrant
|
why += "; " + warrant
|
||||||
}
|
}
|
||||||
if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight}, true, true, held.Revision); err != nil {
|
if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight && w.IsInsight()}, true, true, held.Revision); err != nil {
|
||||||
return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err)
|
return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err)
|
||||||
}
|
}
|
||||||
k.emit(EventAccepted, r, by, why, held.Versions+1)
|
k.emit(EventAccepted, r, by, why, held.Versions+1)
|
||||||
|
|||||||
@@ -917,7 +917,7 @@ func Tools(k *KB, caller func(map[string]any) string) []stdio.Tool {
|
|||||||
"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
|
"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
|
||||||
"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
|
"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
|
||||||
"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
|
"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
|
||||||
"proposal is revised by proposing it again; an accepted record changes only by a progressive insight (insight: what " +
|
"proposal is revised by proposing it again; an accepted record (an active research effort excepted, which takes new versions) changes only by a progressive insight (insight: what " +
|
||||||
"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
|
"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
|
||||||
"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
|
"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
|
||||||
"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +
|
"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +
|
||||||
|
|||||||
@@ -192,7 +192,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
|
|||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
||||||
}
|
}
|
||||||
if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
|
||||||
e.Pending == nil, false, e.Revision); err != nil {
|
e.Pending == nil, false, e.Revision); err != nil {
|
||||||
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
|
||||||
}
|
}
|
||||||
@@ -208,7 +208,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
|
|||||||
return unasked(fmt.Sprintf("the ask could not be made (%v)", err))
|
return unasked(fmt.Sprintf("the ask could not be made (%v)", err))
|
||||||
}
|
}
|
||||||
w.Ask = oa
|
w.Ask = oa
|
||||||
if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil && held.Pending.IsInsight()},
|
||||||
held.Pending == nil, false, held.Revision); err != nil {
|
held.Pending == nil, false, held.Revision); err != nil {
|
||||||
return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err)
|
return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err)
|
||||||
}
|
}
|
||||||
@@ -267,7 +267,7 @@ func (k *KB) Reconcile() ([]string, error) {
|
|||||||
func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string {
|
func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string {
|
||||||
w = w.Clone()
|
w = w.Clone()
|
||||||
w.Ask.Said = why
|
w.Ask.Said = why
|
||||||
if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil},
|
if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
|
||||||
e.Pending == nil, false, e.Revision); err != nil {
|
e.Pending == nil, false, e.Revision); err != nil {
|
||||||
return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err)
|
return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -360,6 +360,10 @@ type PendingState struct {
|
|||||||
Insight Insight `json:"insight"`
|
Insight Insight `json:"insight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsInsight says the record is waiting as a progressive insight, a correction of fact that says what stood before.
|
||||||
|
// A pending version without one is a new version of an active research effort (novox/hq issue 844).
|
||||||
|
func (r *Record) IsInsight() bool { return r.Pending != nil && r.Pending.Insight.Before != "" }
|
||||||
|
|
||||||
// Key is the record's key.
|
// Key is the record's key.
|
||||||
func (r *Record) Key() Key { return Key{r.Kind, r.Number} }
|
func (r *Record) Key() Key { return Key{r.Kind, r.Number} }
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,7 @@ var reviewSchema = map[string]any{
|
|||||||
func ReviewQuestion(r *Record, linked []*Record) string {
|
func ReviewQuestion(r *Record, linked []*Record) string {
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
what := "a proposed " + r.Kind + " record"
|
what := "a proposed " + r.Kind + " record"
|
||||||
if r.Pending != nil {
|
if r.IsInsight() {
|
||||||
what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before
|
what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before
|
||||||
}
|
}
|
||||||
fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+
|
fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+
|
||||||
@@ -475,7 +475,7 @@ func (k *KB) reviewOne(key Key) (string, error) {
|
|||||||
version = now.PendingVersion
|
version = now.PendingVersion
|
||||||
}
|
}
|
||||||
save := func(r *Record, why string) error {
|
save := func(r *Record, why string) error {
|
||||||
return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil},
|
return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil && now.Pending.IsInsight()},
|
||||||
now.Pending == nil, false, now.Revision)
|
now.Pending == nil, false, now.Revision)
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -123,9 +123,16 @@ func allText(r *Record) string {
|
|||||||
// JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the
|
// JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the
|
||||||
// proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
|
// proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
|
||||||
func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
|
func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
|
||||||
|
return JudgeRecordOn(r, held, lookup, ref, r.Date)
|
||||||
|
}
|
||||||
|
|
||||||
|
// JudgeRecordOn is JudgeRecord with the day the dated rules are held from: the record's own day for a new
|
||||||
|
// record, the day of the proposal for a new version of an active research effort (novox/hq issue 844). The rules
|
||||||
|
// that read the record's date (the date and insight-form rules) keep reading r.Date.
|
||||||
|
func JudgeRecordOn(r *Record, held *Entry, lookup Lookup, ref *Reference, day string) []Refusal {
|
||||||
var out []Refusal
|
var out []Refusal
|
||||||
refuse := func(rule, format string, a ...any) {
|
refuse := func(rule, format string, a ...any) {
|
||||||
if Holds(rule, r.Kind, r.Date) {
|
if Holds(rule, r.Kind, day) {
|
||||||
out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
|
out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# mesh-mail-notifier — the default holder of mail-notifier
|
||||||
|
|
||||||
|
Built from mesh-mail's source (novox/hq ADR 0378 §9, as amended by ADR 0379, issue 800 part B): it carries a
|
||||||
|
byte-identical copy of mesh-mail's event shape (`mailevent.go`, pinned by `copies_test.go`, the way
|
||||||
|
`modules/networkmanager`'s `uplink-tools` carries `ifwatchd`'s own file, ADR 0241). Nothing here names
|
||||||
|
Betterbird (ADR 0379 §3), and nothing here reaches a mailbox: it only hears what mesh-mail already said.
|
||||||
|
|
||||||
|
## What it does
|
||||||
|
|
||||||
|
It claims `mail-notifier` (held once per machine, needing `node-notifier` there) and shows each new inbox mail
|
||||||
|
on this machine's notifier, through `node-notifier.send`, at most once:
|
||||||
|
|
||||||
|
- quiet during a first sync or a resync (mesh-mail already says nothing then, `imap.go` ~387)
|
||||||
|
- quiet for a mail from the account's own address (its address, from `mail-store.accounts`; not known, the
|
||||||
|
notice is shown)
|
||||||
|
- quiet for a mail already marked read (`\Seen` in its flags)
|
||||||
|
- quiet for anything outside the inbox
|
||||||
|
- of one account's qualifying mail within a sliding minute, the first three are shown one by one; the rest of
|
||||||
|
that minute's mail is folded into one notice naming the count and the latest sender, rather than shown one
|
||||||
|
by one
|
||||||
|
|
||||||
|
A notice `node-notifier.send` refuses stays pending and is tried again, at most once a minute, without holding
|
||||||
|
up any other account's notice; nothing is deleted before it is shown. Each notice shown emits this module's
|
||||||
|
own event `shown` (account and message id, never sender or subject), and is kept in its own state (`shown`) so
|
||||||
|
a redelivered event is never counted or shown twice — by the mail's message id wherever it has one, so a
|
||||||
|
UIDVALIDITY reset cannot make an already-shown mail look new.
|
||||||
|
|
||||||
|
## Where it runs
|
||||||
|
|
||||||
|
**Its own account** (`runs-as`), because it hears mesh-mail's restricted stream `MAIL` (novox/hq ADR 0376 §2):
|
||||||
|
the machine's runtime runs as the operator's account, which every agent shares and which never hears a
|
||||||
|
restricted stream. Held once per machine (wherever `mail-notifier` is assigned); the operator grants it the
|
||||||
|
accounts it announces, in mesh-mail's trusted setting `grants` (`mail-store`'s stream).
|
||||||
|
|
||||||
|
## Tools
|
||||||
|
|
||||||
|
| tool | does |
|
||||||
|
|---|---|
|
||||||
|
| `mail-notifier.status` | every account it is granted, when it last showed each one a notice, and any notice still failing to send (its count, last error, since when). Read-only |
|
||||||
|
| `mail_notifier_health` | how it is: unhealthy once a notice has failed to show for more than ten minutes, healthy otherwise, even mid-retry. Not (yet) the node-engine's own check — this is one process, judged only by its own tool (ADR 0227 rule 8); a watching process of its own is issue 841, not yet decided. Read-only |
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier carries a byte-identical copy of mesh-mail's event shape (novox/hq ADR 0378 §9, issue 800
|
||||||
|
// part B), the way modules/networkmanager's uplink-tools carries ifwatchd's own file (ADR 0241, copies_test.go):
|
||||||
|
// each builds alone, so each holds its own copy, and this test holds them to one text.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// source is where mesh-mail's own copy of the file lives.
|
||||||
|
var source = filepath.Join("..", "..", "..", "mesh-mail", "cmd", "mesh-mail")
|
||||||
|
|
||||||
|
func TestCarriesTheSameMailEventAsMeshMail(t *testing.T) {
|
||||||
|
for _, f := range []string{"mailevent.go"} {
|
||||||
|
mine, err := os.ReadFile(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
theirs, err := os.ReadFile(filepath.Join(source, f))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mesh-mail's copy of %s could not be read: %v", f, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(mine, theirs) {
|
||||||
|
t.Errorf("this copy of %s differs from mesh-mail's: change both together", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier's own health (review round 1 of issue 800, which added this tool beside mail-notifier's
|
||||||
|
// seat verb status, as mesh-mail's own health tool is mail_health beside its seat tools). A notice that fails
|
||||||
|
// to send stays pending and is retried, at most once a minute (Flush, notifier.go); the account's
|
||||||
|
// failing-since stays in the Store, so this tool — its own process, asked separately by the node-engine or by
|
||||||
|
// mesh_call — sees the same account the live notifier does, with no state of its own to lose between asks.
|
||||||
|
//
|
||||||
|
// Unhealthy once an account's notice has failed to show for more than UnhealthyAfter; healthy otherwise, even
|
||||||
|
// while a notice is mid-retry.
|
||||||
|
//
|
||||||
|
// STOP (review round 1 of issue 800, not decided by its note): this tool is served and tested, but module.json
|
||||||
|
// does not name it as the notify process's own health check — mesh-mail-notifier has one process (notify), and
|
||||||
|
// the catalogue's gate refuses a module judged only by its own tool, with no check of another kind beside it
|
||||||
|
// (ADR 0227 rule 8, ADR 0240 rule 2; internal/catalogue/health.go ~297 in mesh-controller). mesh-mail avoids
|
||||||
|
// this because it has two processes — keep (kind unit) beside watch (kind tool, mail_health); mesh-mail-notifier
|
||||||
|
// has only notify, so wiring this tool the same way trips the gate. notify's health stays kind unit (liveness,
|
||||||
|
// as it was before this round) until the lead decides how a one-process module gets an automatic, node-engine
|
||||||
|
// enforced health of this kind: a second process judged some other way, or a change to the gate's rule.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthTool is the module's own health tool the node-engine asks (beside mail-notifier.status, the seat's own verb).
|
||||||
|
const HealthTool = "mail_notifier_health"
|
||||||
|
|
||||||
|
// UnhealthyAfter is how long a notice may fail to send before that is unhealthy, not merely retrying.
|
||||||
|
const UnhealthyAfter = 10 * time.Minute
|
||||||
|
|
||||||
|
// Judge is mesh-mail-notifier's health, from the failures the Store keeps.
|
||||||
|
func Judge(failures map[string]FailureStatus, now time.Time) health.Answer {
|
||||||
|
if len(failures) == 0 {
|
||||||
|
return health.Healthy("every notice shown; none pending a retry")
|
||||||
|
}
|
||||||
|
accounts := make([]string, 0, len(failures))
|
||||||
|
for account := range failures {
|
||||||
|
accounts = append(accounts, account)
|
||||||
|
}
|
||||||
|
sort.Strings(accounts)
|
||||||
|
|
||||||
|
var problems, retrying []string
|
||||||
|
for _, account := range accounts {
|
||||||
|
f := failures[account]
|
||||||
|
age := now.Sub(f.Since).Round(time.Second)
|
||||||
|
if now.Sub(f.Since) > UnhealthyAfter {
|
||||||
|
problems = append(problems, fmt.Sprintf("%s: its notice has failed to show for %s (%d attempt(s), last: %s)",
|
||||||
|
account, age, f.Count, f.LastError))
|
||||||
|
} else {
|
||||||
|
retrying = append(retrying, fmt.Sprintf("%s has been retrying for %s", account, age))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
return health.Unhealthy(strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
return health.Healthy(strings.Join(retrying, "; "))
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJudgeIsUnhealthyOnlyPastUnhealthyAfter(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
if a := Judge(nil, now); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("no failures: %+v, want healthy", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
recent := map[string]FailureStatus{"work": {Count: 1, LastError: "no answer", Since: now.Add(-2 * time.Minute)}}
|
||||||
|
if a := Judge(recent, now); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("a failure 2m old: %+v, want healthy (still retrying)", a)
|
||||||
|
}
|
||||||
|
|
||||||
|
stale := map[string]FailureStatus{"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)}}
|
||||||
|
a := Judge(stale, now)
|
||||||
|
if a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("a failure 11m old: %+v, want unhealthy (past UnhealthyAfter)", a)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Why, "work") || !strings.Contains(a.Why, "no answer") {
|
||||||
|
t.Errorf("Why = %q, want it to name the account and the last error", a.Why)
|
||||||
|
}
|
||||||
|
|
||||||
|
mixed := map[string]FailureStatus{
|
||||||
|
"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)},
|
||||||
|
"home": {Count: 1, LastError: "no answer", Since: now.Add(-time.Minute)},
|
||||||
|
}
|
||||||
|
if a := Judge(mixed, now); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "work") {
|
||||||
|
t.Errorf("one stale failure beside one fresh one: %+v, want unhealthy naming work", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The shape of a mail-store message event (novox/hq ADR 0378 §2, issue 800 part A), moved here on its own so
|
||||||
|
// mesh-mail-notifier (issue 800 part B) carries a byte-identical copy, the way modules/networkmanager's
|
||||||
|
// uplink-tools carries ifwatchd's own file (novox/hq ADR 0241, copies_test.go). Nothing here reaches a mailbox or
|
||||||
|
// a server; it only says where an account's change is told on the bus and what the three changes are called.
|
||||||
|
|
||||||
|
// EventKey is where an account's change is said: mail.<account>.message.<added|changed|removed> (research 049
|
||||||
|
// document 02 §5), the account by its name in the setting, never its address.
|
||||||
|
func EventKey(account, change string) string { return "mail." + account + ".message." + change }
|
||||||
|
|
||||||
|
// Changes are the three things said of a message.
|
||||||
|
var Changes = []string{"added", "changed", "removed"}
|
||||||
@@ -0,0 +1,232 @@
|
|||||||
|
// mesh-mail-notifier: the default claimant of mail-notifier, built from mesh-mail's own event shape (novox/hq
|
||||||
|
// ADR 0378 §9, issue 800 part B). A Go bundle the node's runtime launches, its own account (runs-as, ADR 0376
|
||||||
|
// §2): the one process serves mail-notifier.status and the module's own health tool (mail_notifier_health,
|
||||||
|
// review round 1 of issue 800), and, beside stdio.Serve, hears mesh-mail's added events and shows them
|
||||||
|
// (exactly as modules/mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve) — not
|
||||||
|
// a separate `notify` process, which systemd gave an empty stdin and so exited at once, every five seconds
|
||||||
|
// (novox/hq issue 859). What this module says, it says on stderr; stdout is the MCP channel.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
|
||||||
|
|
||||||
|
// AddedPattern is the mesh-mail stream's event this module consumes (its own `consumes` entry, novox/hq ADR
|
||||||
|
// 0363, ADR 0378 §9): a new, changed or removed message's `added` only.
|
||||||
|
const AddedPattern = "mesh-mail.mail.*.message.added"
|
||||||
|
|
||||||
|
// StatusTool is the mail-notifier seat's one verb this module serves.
|
||||||
|
const StatusTool = "mail-notifier.status"
|
||||||
|
|
||||||
|
// ShownEvent is this module's own event (its `emits`), one per notice shown.
|
||||||
|
const ShownEvent = "shown"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
n := &Notifier{
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
_, err := stdio.Ask(sendKey(), args)
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
Accounts: readAccounts,
|
||||||
|
Store: busStore{},
|
||||||
|
EmitShown: func(account, messageID string) error {
|
||||||
|
return stdio.Emit(ShownEvent, map[string]any{"account": account, "message-id": messageID})
|
||||||
|
},
|
||||||
|
Logf: logf,
|
||||||
|
}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
tools := []stdio.Tool{
|
||||||
|
{Name: StatusTool,
|
||||||
|
Description: "The accounts mesh-mail-notifier announces on this machine, and when it last showed a " +
|
||||||
|
"notice of new mail (novox/hq ADR 0378 §7). Read-only.",
|
||||||
|
Run: func(map[string]any) (any, error) { return n.Status() }},
|
||||||
|
{Name: HealthTool,
|
||||||
|
Description: "How mesh-mail-notifier is: unhealthy once a notice has failed to show for more than " +
|
||||||
|
"ten minutes, healthy otherwise, even mid-retry (review round 1 of issue 800). Read-only.",
|
||||||
|
Run: func(map[string]any) (any, error) {
|
||||||
|
failures, err := busStore{}.Failures()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return Judge(failures, time.Now()), nil
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
if err := run(n, watch, os.Stdin, os.Stdout, tools); err != nil {
|
||||||
|
logf("%v", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendKey is the ask key Send uses: this machine's own node-notifier, named exactly as desk-channel asks
|
||||||
|
// another machine's ("seat:node-notifier.send@"+machine) and as the SDK's own Emit reads this machine's name
|
||||||
|
// (MESH_NODE, which the runtime sets) — not the bare seat, which any machine's node-notifier may answer
|
||||||
|
// first, leaving this one's notices undelivered (novox/hq issue 865).
|
||||||
|
func sendKey() string { return "seat:node-notifier.send@" + os.Getenv("MESH_NODE") }
|
||||||
|
|
||||||
|
// run is main's startup path, factored so a test can run it: the watcher started before stdio.Serve, exactly
|
||||||
|
// as mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve — so the watcher keeps
|
||||||
|
// running whatever stdio.Serve does with the stream it is given, including returning at once when stdin is
|
||||||
|
// already at EOF, as systemd gave the old separate `notify` process (novox/hq issue 859). The node's runtime
|
||||||
|
// launches this one process and keeps it alive.
|
||||||
|
func run(n *Notifier, watcher func(*Notifier), in io.Reader, out io.Writer, tools []stdio.Tool) error {
|
||||||
|
go watcher(n)
|
||||||
|
return stdio.ServeOn("", tools, in, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// watch hears mesh-mail's added events and, once a second, flushes what Notifier.Flush finds ready: an
|
||||||
|
// account's first mails of its window at once, and its folded overflow once the window is over (ADR 0378 §9,
|
||||||
|
// the lead's reading, review round 1 of issue 800).
|
||||||
|
func watch(n *Notifier) { watchWith(n, stdio.Subscribe, time.Sleep) }
|
||||||
|
|
||||||
|
// watchWith is watch with its subscribe and sleep factored out, so a test can make a subscribe fail without
|
||||||
|
// the real bus and without the real wait. A subscribe the runtime refuses is retried, 2 seconds doubling to
|
||||||
|
// at most a minute, logging each refusal — exactly as modules/mesh-knowledge's hear does — never fatal: a
|
||||||
|
// refusal here must never take mail-notifier.status and mail_notifier_health down with it (novox/hq issue
|
||||||
|
// 859, review round 1; the old os.Exit(1) did exactly that).
|
||||||
|
func watchWith(n *Notifier, subscribe func(string, func(stdio.Envelope) error) error, sleep func(time.Duration)) {
|
||||||
|
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||||
|
sleep(wait)
|
||||||
|
err := subscribe(AddedPattern, func(e stdio.Envelope) error {
|
||||||
|
var body EventBody
|
||||||
|
if err := json.Unmarshal(e.Body, &body); err != nil {
|
||||||
|
logf("[mesh-mail-notifier] an added event could not be read; taken and dropped: %v", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return n.Add(body)
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
logf("[mesh-mail-notifier] does not hear %s yet (%v); asking again in %s", AddedPattern, err, wait)
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(1 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for range ticker.C {
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
logf("[mesh-mail-notifier] a buffered notice could not be shown: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readAccounts asks mail-store.accounts for the accounts this module is granted, by name to own address.
|
||||||
|
func readAccounts() (map[string]string, error) {
|
||||||
|
raw, err := stdio.Ask("seat:mail-store.accounts", map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var answer struct {
|
||||||
|
Accounts []struct {
|
||||||
|
Account string `json:"account"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
} `json:"accounts"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, a := range answer.Accounts {
|
||||||
|
out[a.Account] = a.Address
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busStore keeps the notifier's own state (the `shown` bucket the manifest declares): which messages were
|
||||||
|
// shown already, each account's last notice, and each account's notice still failing to send.
|
||||||
|
type busStore struct{}
|
||||||
|
|
||||||
|
func stateKey(parts ...string) string {
|
||||||
|
words := make([]string, len(parts))
|
||||||
|
for i, p := range parts {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range p {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-':
|
||||||
|
b.WriteRune(r)
|
||||||
|
default:
|
||||||
|
b.WriteRune('_')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words[i] = b.String()
|
||||||
|
}
|
||||||
|
return strings.Join(words, "_")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) HandleSeen(handle string) (bool, error) {
|
||||||
|
e, err := stdio.State("shown").Get("handle_" + stateKey(strings.Split(handle, "\x1f")...))
|
||||||
|
return e != nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) MarkHandle(handle string) error {
|
||||||
|
_, err := stdio.State("shown").Put("handle_"+stateKey(strings.Split(handle, "\x1f")...), true)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) PutStatus(account string, s AccountStatus) error {
|
||||||
|
_, err := stdio.State("shown").Put("status_"+stateKey(account), s)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) Statuses() (map[string]AccountStatus, error) {
|
||||||
|
keys, err := stdio.State("shown").Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]AccountStatus{}
|
||||||
|
for _, k := range keys {
|
||||||
|
account, ok := strings.CutPrefix(k, "status_")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := stdio.State("shown").Get(k)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var s AccountStatus
|
||||||
|
if json.Unmarshal(e.Value, &s) == nil {
|
||||||
|
out[account] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) PutFailure(account string, f FailureStatus) error {
|
||||||
|
_, err := stdio.State("shown").Put("failure_"+stateKey(account), f)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) ClearFailure(account string) error {
|
||||||
|
return stdio.State("shown").Delete("failure_" + stateKey(account))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (busStore) Failures() (map[string]FailureStatus, error) {
|
||||||
|
keys, err := stdio.State("shown").Keys()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]FailureStatus{}
|
||||||
|
for _, k := range keys {
|
||||||
|
account, ok := strings.CutPrefix(k, "failure_")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e, err := stdio.State("shown").Get(k)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var f FailureStatus
|
||||||
|
if json.Unmarshal(e.Value, &f) == nil {
|
||||||
|
out[account] = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The manifest's own shape (novox/hq issue 859): the watch loop runs in the base invocation beside
|
||||||
|
// stdio.Serve, so no resource declares a separate `notify` process any more — the thing that exited at once,
|
||||||
|
// every five seconds, because a systemd-run process's stdin is empty and stdio.Serve returns at EOF.
|
||||||
|
|
||||||
|
type manifestResources struct {
|
||||||
|
Resources []map[string]any `json:"resources"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func readManifestResources(t *testing.T) manifestResources {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var m manifestResources
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
t.Fatalf("module.json: %v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoProcessResourceIsDeclared(t *testing.T) {
|
||||||
|
m := readManifestResources(t)
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["type"] == "process" {
|
||||||
|
t.Fatalf("module.json still declares a process resource (%v): the watcher runs in the base invocation now, not a separate process (novox/hq issue 859)", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,537 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mesh-mail-notifier: the default holder of mail-notifier (novox/hq ADR 0378 §7, as amended by ADR 0379 §4,
|
||||||
|
// issue 800 part B). It hears mesh-mail's added events on its own account (runs-as, ADR 0376 §2) and shows each
|
||||||
|
// new inbox mail at most once, through node-notifier.send: quiet during a first sync (mesh-mail already says
|
||||||
|
// nothing then, imap.go ~387), for mail from the account's own address, for mail already marked read (\Seen),
|
||||||
|
// and for anything outside the inbox. For each new qualifying mail of an account, arriving at t, N counts that
|
||||||
|
// account's qualifying mail — shown or folded alike, each counted once — that arrived in the 60 seconds up to
|
||||||
|
// and including t: N <= FoldOver is shown at once; N > FoldOver is folded. The first folded mail opens the
|
||||||
|
// account's fold; every folded mail joins it; the fold is shown, as one notice naming how many it holds and
|
||||||
|
// the latest sender, once Window has passed since its own first mail — after which a new fold may open. The
|
||||||
|
// count is a true sliding window of arrivals, not a per-session quota that only resets after a quiet Window:
|
||||||
|
// review round 2's reading kept a session's count alive, and so folding, for as long as any gap stayed under
|
||||||
|
// Window, which folds a steady trickle (one mail every 50s, say) forever even though no burst ever happened;
|
||||||
|
// review round 3 of issue 800 replaces that reading with this one. Nothing pending — a folded mail awaiting
|
||||||
|
// its fold's deadline, or a failed send awaiting its retry — is ever dropped (novox/hq ADR 0378 §9, review
|
||||||
|
// round 3 of issue 800, which replaces the round 1 and round 2 readings).
|
||||||
|
//
|
||||||
|
// Nothing here names Betterbird (ADR 0379 §3).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Inbox is the one folder a new mail is shown for.
|
||||||
|
const Inbox = "INBOX"
|
||||||
|
|
||||||
|
// Seen is the flag that means a mail was already read: shown for nothing it marks.
|
||||||
|
const SeenFlag = "\\Seen"
|
||||||
|
|
||||||
|
// FoldOver is how many new mails from one account, within one window, are still shown one by one; the rest of
|
||||||
|
// that window is folded into a single notice (novox/hq ADR 0378 §9: "more than three").
|
||||||
|
const FoldOver = 3
|
||||||
|
|
||||||
|
// Window is the trailing 60 seconds ADR 0378 §9 counts a mail's N over (every qualifying mail of the account,
|
||||||
|
// shown or folded alike, that arrived in the 60 seconds up to and including this one), and the time a fold
|
||||||
|
// waits, from its own first mail, before it is shown (review round 3 of issue 800, which replaces the round 1
|
||||||
|
// and round 2 readings: the count is a true sliding window of arrivals, not a per-session quota that stays
|
||||||
|
// alive, and so keeps folding, for as long as any gap between mails stays under Window).
|
||||||
|
const Window = time.Minute
|
||||||
|
|
||||||
|
// RetryBackoff is how long the notifier waits, after an account's notice fails to send, before it tries that
|
||||||
|
// account again: at most once a minute (review round 1 of issue 800), so a send that keeps failing is not
|
||||||
|
// retried on every Flush.
|
||||||
|
const RetryBackoff = time.Minute
|
||||||
|
|
||||||
|
// AccountsCacheFor is how long mail-store.accounts's answer is reused before it is asked again (review round
|
||||||
|
// 1 of issue 800, bullet 5): neither the quiet rule nor status asks it more than once a minute.
|
||||||
|
const AccountsCacheFor = time.Minute
|
||||||
|
|
||||||
|
// Person is a sender or a recipient, as mesh-mail's event says it.
|
||||||
|
type Person struct {
|
||||||
|
Name string `json:"name,omitempty"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// EventBody is what mesh-mail's added event says of a message (metadata only; mailevent.go names where it is
|
||||||
|
// said). Unknown fields are ignored: the notifier reads only what it needs.
|
||||||
|
type EventBody struct {
|
||||||
|
Account string `json:"account"`
|
||||||
|
Folder string `json:"folder"`
|
||||||
|
UID uint32 `json:"uid"`
|
||||||
|
UIDValidity uint32 `json:"uidvalidity"`
|
||||||
|
MessageID string `json:"message-id"`
|
||||||
|
Subject string `json:"subject"`
|
||||||
|
From []Person `json:"from"`
|
||||||
|
Flags []string `json:"flags"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle names one message, stably, for the dedupe state (novox/hq ADR 0378 §9: "a redelivered event is not
|
||||||
|
// shown twice"). The message id is the key wherever a message carries one, so a UIDVALIDITY reset — which
|
||||||
|
// changes every UID in a folder — can never make an already-shown mail look new, or a genuinely new one look
|
||||||
|
// already shown; only a message with no message id falls back to its folder, UIDVALIDITY and UID (review
|
||||||
|
// round 1 of issue 800).
|
||||||
|
func (e EventBody) Handle() string {
|
||||||
|
if e.MessageID != "" {
|
||||||
|
return e.Account + "\x1f" + e.MessageID
|
||||||
|
}
|
||||||
|
return e.Account + "\x1f" + e.Folder + "\x1f" +
|
||||||
|
strconv.FormatUint(uint64(e.UIDValidity), 10) + "\x1f" + strconv.FormatUint(uint64(e.UID), 10)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e EventBody) hasFlag(flag string) bool {
|
||||||
|
for _, f := range e.Flags {
|
||||||
|
if strings.EqualFold(f, flag) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sender is how a message's first sender reads in a notice: its name and address, or its address alone.
|
||||||
|
func (e EventBody) sender() string {
|
||||||
|
if len(e.From) == 0 {
|
||||||
|
return "(unknown sender)"
|
||||||
|
}
|
||||||
|
p := e.From[0]
|
||||||
|
if p.Name == "" {
|
||||||
|
return p.Address
|
||||||
|
}
|
||||||
|
return p.Name + " <" + p.Address + ">"
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e EventBody) fromAddress() string {
|
||||||
|
if len(e.From) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return e.From[0].Address
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notice is one notification the notifier shows: its args for node-notifier.send, and the message its `shown`
|
||||||
|
// event names (the latest one, for a folded notice).
|
||||||
|
type Notice struct {
|
||||||
|
Account string
|
||||||
|
Args map[string]any
|
||||||
|
Handles []string // every message this notice accounts for, marked seen once shown
|
||||||
|
Latest EventBody
|
||||||
|
}
|
||||||
|
|
||||||
|
// FailureStatus is what the notifier knows of an account's notice still failing to send (review round 1 of
|
||||||
|
// issue 800): how many attempts, the last error, and since when. Kept in the Store, not only in memory, so
|
||||||
|
// the health tool — its own process, asked separately by the node-engine — sees the same account the live
|
||||||
|
// notifier does.
|
||||||
|
type FailureStatus struct {
|
||||||
|
Count int `json:"count"`
|
||||||
|
LastError string `json:"last-error"`
|
||||||
|
Since time.Time `json:"since"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store is what the notifier keeps of its own (the `shown` state the manifest declares): whether a message
|
||||||
|
// was shown already, each account's last notice, and each account's notice still failing to send.
|
||||||
|
type Store interface {
|
||||||
|
HandleSeen(handle string) (bool, error)
|
||||||
|
MarkHandle(handle string) error
|
||||||
|
PutStatus(account string, s AccountStatus) error
|
||||||
|
Statuses() (map[string]AccountStatus, error)
|
||||||
|
PutFailure(account string, f FailureStatus) error
|
||||||
|
ClearFailure(account string) error
|
||||||
|
Failures() (map[string]FailureStatus, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccountStatus is what the notifier last showed of one account.
|
||||||
|
type AccountStatus struct {
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// accountBuffer is one account's state: history is every qualifying mail's arrival time, kept only for the
|
||||||
|
// trailing Window, that N is counted against (review round 3 of issue 800); immediate holds mail waiting to be
|
||||||
|
// shown one by one at the next Flush; folded holds the account's current fold (its first FoldOver mails within
|
||||||
|
// Window are never folded, so a fold only ever holds the overflow from the fourth qualifying mail of its own
|
||||||
|
// window onward), due as one notice once Window has passed since foldStart, its first mail's arrival.
|
||||||
|
type accountBuffer struct {
|
||||||
|
history []time.Time
|
||||||
|
foldStart time.Time // when the current fold's first mail arrived; zero while nothing is folded
|
||||||
|
immediate []EventBody
|
||||||
|
folded []EventBody
|
||||||
|
}
|
||||||
|
|
||||||
|
// Notifier buffers qualifying mail per account and shows it: the first FoldOver mails of a window one by one,
|
||||||
|
// the rest folded once the window ends. A notice that fails to send is tried again, at most once a minute,
|
||||||
|
// without holding up any other account's notice.
|
||||||
|
type Notifier struct {
|
||||||
|
// Send calls node-notifier.send on this machine.
|
||||||
|
Send func(args map[string]any) error
|
||||||
|
// Accounts reads the accounts the notifier is granted, by mail-store.accounts: each one's own address, for
|
||||||
|
// the quiet rule (novox/hq ADR 0378 §9's LEAD note). An account not known there is shown anyway.
|
||||||
|
Accounts func() (map[string]string, error)
|
||||||
|
Store Store
|
||||||
|
// EmitShown says mesh-mail-notifier's own event `shown`: account and message id, never sender or subject.
|
||||||
|
EmitShown func(account, messageID string) error
|
||||||
|
Now func() time.Time
|
||||||
|
Logf func(string, ...any)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
pending map[string]*accountBuffer
|
||||||
|
buffered map[string]bool // a message's Handle while it is waiting to be shown: never counted twice
|
||||||
|
cooldown map[string]time.Time // account -> not retried before this time, after a failed send
|
||||||
|
|
||||||
|
accountsAt time.Time
|
||||||
|
accountsVal map[string]string
|
||||||
|
accountsErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) init() {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
if n.pending == nil {
|
||||||
|
n.pending = map[string]*accountBuffer{}
|
||||||
|
n.buffered = map[string]bool{}
|
||||||
|
n.cooldown = map[string]time.Time{}
|
||||||
|
}
|
||||||
|
if n.Now == nil {
|
||||||
|
n.Now = time.Now
|
||||||
|
}
|
||||||
|
if n.Logf == nil {
|
||||||
|
n.Logf = func(string, ...any) {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add takes one added event. It returns an error only to have the event offered again (the handler
|
||||||
|
// Subscribe wraps this in follows that rule).
|
||||||
|
func (n *Notifier) Add(e EventBody) error {
|
||||||
|
n.init()
|
||||||
|
handle := e.Handle()
|
||||||
|
seen, err := n.Store.HandleSeen(handle)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if seen {
|
||||||
|
return nil // a redelivery of a message already shown (or already judged not to be)
|
||||||
|
}
|
||||||
|
own, known, err := n.ownAddress(e.Account)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !n.qualifies(e, own, known) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
if n.buffered[handle] {
|
||||||
|
return nil // a redelivery of a message still waiting to be shown: never counted twice
|
||||||
|
}
|
||||||
|
now := n.Now()
|
||||||
|
b := n.pending[e.Account]
|
||||||
|
if b == nil {
|
||||||
|
b = &accountBuffer{}
|
||||||
|
n.pending[e.Account] = b
|
||||||
|
}
|
||||||
|
// N is every qualifying mail of this account — shown or folded alike, each counted once — that arrived in
|
||||||
|
// the 60 seconds up to and including now: a true sliding window over arrivals, not a per-session quota that
|
||||||
|
// stays alive, and so keeps folding, for as long as any gap stays under Window (review round 3 of issue
|
||||||
|
// 800, which replaces the round 1 and round 2 readings; nothing in this count depends on whether an earlier
|
||||||
|
// mail has been shown yet).
|
||||||
|
cutoff := now.Add(-Window)
|
||||||
|
history := make([]time.Time, 0, len(b.history)+1)
|
||||||
|
for _, at := range b.history {
|
||||||
|
if !at.Before(cutoff) {
|
||||||
|
history = append(history, at)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.history = append(history, now)
|
||||||
|
n.buffered[handle] = true
|
||||||
|
if len(b.history) <= FoldOver {
|
||||||
|
b.immediate = append(b.immediate, e)
|
||||||
|
} else {
|
||||||
|
if len(b.folded) == 0 {
|
||||||
|
b.foldStart = now
|
||||||
|
}
|
||||||
|
b.folded = append(b.folded, e)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// qualifies says whether a mail is shown at all: a new inbox mail, not the account's own, not already read.
|
||||||
|
func (n *Notifier) qualifies(e EventBody, ownAddress string, ownKnown bool) bool {
|
||||||
|
if !strings.EqualFold(e.Folder, Inbox) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if e.hasFlag(SeenFlag) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if ownKnown && ownAddress != "" && strings.EqualFold(e.fromAddress(), ownAddress) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) ownAddress(account string) (string, bool, error) {
|
||||||
|
accounts, err := n.readAccounts()
|
||||||
|
if err != nil {
|
||||||
|
// Not known: ADR 0378 §9's LEAD note says the notice is shown when the address cannot be read. Logged,
|
||||||
|
// not retried: mail-store being unreachable is not this event's fault.
|
||||||
|
n.Logf("[mesh-mail-notifier] the account's own address could not be read (%v); the notice is shown", err)
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
addr, ok := accounts[account]
|
||||||
|
return addr, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readAccounts asks mail-store.accounts, reusing its last answer for AccountsCacheFor so neither the quiet
|
||||||
|
// rule nor status reads it more than once a minute (review round 1 of issue 800, bullet 5).
|
||||||
|
func (n *Notifier) readAccounts() (map[string]string, error) {
|
||||||
|
n.mu.Lock()
|
||||||
|
if n.accountsVal != nil && n.Now().Sub(n.accountsAt) < AccountsCacheFor {
|
||||||
|
val, err := n.accountsVal, n.accountsErr
|
||||||
|
n.mu.Unlock()
|
||||||
|
return val, err
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
|
||||||
|
val, err := n.Accounts()
|
||||||
|
n.mu.Lock()
|
||||||
|
n.accountsVal, n.accountsErr, n.accountsAt = val, err, n.Now()
|
||||||
|
n.mu.Unlock()
|
||||||
|
return val, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush shows what is ready: each account's first FoldOver mails of its window, one by one, as soon as Flush
|
||||||
|
// runs, and the window's folded overflow once the window has run its minute. Accounts are judged in a sorted
|
||||||
|
// order, not a map's iteration order, so the tests do not depend on it (review round 1 of issue 800, BLOCKING
|
||||||
|
// 1). A notice that fails to send leaves its mail pending, tried again once RetryBackoff has passed, and never
|
||||||
|
// holds up another account's notice (BLOCKING 2): nothing is deleted before it is shown.
|
||||||
|
func (n *Notifier) Flush() error {
|
||||||
|
n.init()
|
||||||
|
now := n.Now()
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
accounts := make([]string, 0, len(n.pending))
|
||||||
|
for account := range n.pending {
|
||||||
|
accounts = append(accounts, account)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
sort.Strings(accounts)
|
||||||
|
|
||||||
|
for _, account := range accounts {
|
||||||
|
n.flushAccount(account, now)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// flushAccount ships one account's ready notices (its immediate mail, and its fold once Window has passed
|
||||||
|
// since the fold's own first mail), and keeps whatever fails to send pending, under cooldown, for the next
|
||||||
|
// Flush. Nothing is deleted before it is shown (review round 1 BLOCKING 2), and nothing pending is ever lost
|
||||||
|
// (review round 2 BLOCKING, kept under review round 3's rule; see Add).
|
||||||
|
func (n *Notifier) flushAccount(account string, now time.Time) {
|
||||||
|
n.mu.Lock()
|
||||||
|
if cd, ok := n.cooldown[account]; ok && now.Before(cd) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
return // backing off after a failed send; tried again once RetryBackoff has passed
|
||||||
|
}
|
||||||
|
b := n.pending[account]
|
||||||
|
if b == nil {
|
||||||
|
n.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
immediate := append([]EventBody(nil), b.immediate...)
|
||||||
|
// The fold is due Window after its own first mail (review round 3 of issue 800).
|
||||||
|
foldDue := len(b.folded) > 0 && now.Sub(b.foldStart) >= Window
|
||||||
|
var fold []EventBody
|
||||||
|
if foldDue {
|
||||||
|
fold = append([]EventBody(nil), b.folded...)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
|
||||||
|
if len(immediate) == 0 && len(fold) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
sentImmediate := 0
|
||||||
|
var failErr error
|
||||||
|
for _, e := range immediate {
|
||||||
|
if err := n.show(n.singleNotice(account, e)); err != nil {
|
||||||
|
failErr = err
|
||||||
|
break
|
||||||
|
}
|
||||||
|
sentImmediate++
|
||||||
|
}
|
||||||
|
foldSent := false
|
||||||
|
if failErr == nil && len(fold) > 0 {
|
||||||
|
if err := n.show(n.foldNotice(account, fold, len(fold))); err != nil {
|
||||||
|
failErr = err
|
||||||
|
} else {
|
||||||
|
foldSent = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n.mu.Lock()
|
||||||
|
if cur := n.pending[account]; cur == b {
|
||||||
|
cur.immediate = cur.immediate[sentImmediate:]
|
||||||
|
if foldSent {
|
||||||
|
cur.folded = cur.folded[len(fold):]
|
||||||
|
if len(cur.folded) == 0 {
|
||||||
|
cur.foldStart = time.Time{} // nothing left folded: a new fold may open (review round 3 of issue 800)
|
||||||
|
} else {
|
||||||
|
// More mail folded in while this batch was being sent: its own Window starts now, counted from
|
||||||
|
// when it was first noticed. Nothing of it is lost either way.
|
||||||
|
cur.foldStart = now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failErr != nil {
|
||||||
|
n.cooldown[account] = now.Add(RetryBackoff)
|
||||||
|
n.recordFailureLocked(account, failErr, now)
|
||||||
|
} else {
|
||||||
|
delete(n.cooldown, account)
|
||||||
|
n.clearFailureLocked(account)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordFailureLocked keeps an account's failure in the Store (mu held by the caller, released around the
|
||||||
|
// Store calls since they may reach the bus): its count, its last error, and since when, so a later Flush and
|
||||||
|
// the health tool agree on how long it has been failing.
|
||||||
|
func (n *Notifier) recordFailureLocked(account string, sendErr error, now time.Time) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
defer n.mu.Lock()
|
||||||
|
since := now
|
||||||
|
count := 1
|
||||||
|
if prev, err := n.Store.Failures(); err == nil {
|
||||||
|
if p, ok := prev[account]; ok {
|
||||||
|
since = p.Since
|
||||||
|
count = p.Count + 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Store.PutFailure(account, FailureStatus{Count: count, LastError: sendErr.Error(), Since: since}); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] a failure could not be kept (%v): the health tool may not see it", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (n *Notifier) clearFailureLocked(account string) {
|
||||||
|
n.mu.Unlock()
|
||||||
|
defer n.mu.Lock()
|
||||||
|
if err := n.Store.ClearFailure(account); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] a cleared failure could not be kept cleared: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// singleNotice is one mail shown on its own: one of an account's first FoldOver mails of its window.
|
||||||
|
func (n *Notifier) singleNotice(account string, e EventBody) Notice {
|
||||||
|
return Notice{
|
||||||
|
Account: account,
|
||||||
|
Args: map[string]any{
|
||||||
|
"summary": e.Subject,
|
||||||
|
"body": e.sender() + " — " + account,
|
||||||
|
"app_name": "mesh-mail",
|
||||||
|
},
|
||||||
|
Handles: []string{e.Handle()},
|
||||||
|
Latest: e,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// foldNotice is one notice for everything the account's current fold holds — its qualifying mail from the
|
||||||
|
// fourth within some trailing 60 seconds onward — naming how many mail that fold holds (not the account's
|
||||||
|
// whole history of arrivals) and the latest sender (novox/hq ADR 0378 §9, review round 3 of issue 800, which
|
||||||
|
// replaces the round 1 and round 2 readings: "<k> more new mail(s) in <account>, latest from <sender of the
|
||||||
|
// last>", "mail" for one and "mails" for more).
|
||||||
|
func (n *Notifier) foldNotice(account string, events []EventBody, count int) Notice {
|
||||||
|
latest := events[len(events)-1]
|
||||||
|
notice := Notice{
|
||||||
|
Account: account,
|
||||||
|
Args: map[string]any{
|
||||||
|
"summary": strconv.Itoa(count) + " more new " + mailWord(count) + " in " + account,
|
||||||
|
"body": "latest from " + latest.sender(),
|
||||||
|
"app_name": "mesh-mail",
|
||||||
|
},
|
||||||
|
Latest: latest,
|
||||||
|
}
|
||||||
|
for _, e := range events {
|
||||||
|
notice.Handles = append(notice.Handles, e.Handle())
|
||||||
|
}
|
||||||
|
return notice
|
||||||
|
}
|
||||||
|
|
||||||
|
// mailWord says "mail" for a count of one, "mails" for anything else (review round 2 of issue 800, bullet 3;
|
||||||
|
// kept by review round 3).
|
||||||
|
func mailWord(count int) string {
|
||||||
|
if count == 1 {
|
||||||
|
return "mail"
|
||||||
|
}
|
||||||
|
return "mails"
|
||||||
|
}
|
||||||
|
|
||||||
|
// show sends one notice, marks every message it accounts for seen, keeps the account's status, and emits
|
||||||
|
// `shown`. Nothing is marked, kept or emitted when Send fails: the caller keeps the notice pending instead.
|
||||||
|
func (n *Notifier) show(notice Notice) error {
|
||||||
|
if err := n.Send(notice.Args); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] %s's notice could not be shown: %v", notice.Account, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n.mu.Lock()
|
||||||
|
for _, h := range notice.Handles {
|
||||||
|
delete(n.buffered, h)
|
||||||
|
}
|
||||||
|
n.mu.Unlock()
|
||||||
|
for _, h := range notice.Handles {
|
||||||
|
if err := n.Store.MarkHandle(h); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] shown, but not kept as shown (%v): a redelivery may show it again", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
summary, _ := notice.Args["summary"].(string)
|
||||||
|
if err := n.Store.PutStatus(notice.Account, AccountStatus{At: n.Now(), Summary: summary}); err != nil {
|
||||||
|
n.Logf("[mesh-mail-notifier] shown, but the status could not be kept: %v", err)
|
||||||
|
}
|
||||||
|
return n.EmitShown(notice.Account, notice.Latest.MessageID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is the notifier's account of itself, for mail-notifier.status (novox/hq ADR 0378 §7: "accounts it
|
||||||
|
// announces there, when it last showed a notice"; review round 1 of issue 800: the accounts
|
||||||
|
// mail-store.accounts gives it, and the failures it is retrying).
|
||||||
|
func (n *Notifier) Status() (map[string]any, error) {
|
||||||
|
statuses, err := n.Store.Statuses()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
failures, err := n.Store.Failures()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
known, knownErr := n.readAccounts()
|
||||||
|
accounts := map[string]any{}
|
||||||
|
for account := range known {
|
||||||
|
accounts[account] = map[string]any{}
|
||||||
|
}
|
||||||
|
for account, s := range statuses {
|
||||||
|
accounts[account] = map[string]any{"at": s.At.UTC().Format(time.RFC3339), "shown": s.Summary}
|
||||||
|
}
|
||||||
|
out := map[string]any{
|
||||||
|
"accounts": accounts,
|
||||||
|
"rules": []string{
|
||||||
|
"it shows each new inbox mail at most once, through node-notifier.send on this machine (novox/hq ADR 0378 §7)",
|
||||||
|
"quiet during a first sync, for mail from the account's own address, for mail already marked read, and for anything outside the inbox",
|
||||||
|
"of one account's qualifying mail, one arriving when fewer than three of the account's mails fell in the 60 seconds before it is shown at once; the rest is folded and shown, as one notice naming how many it holds and the latest sender, 60 seconds after the fold's own first mail",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if knownErr != nil {
|
||||||
|
out["accounts-error"] = knownErr.Error()
|
||||||
|
}
|
||||||
|
if len(failures) > 0 {
|
||||||
|
failing := map[string]any{}
|
||||||
|
for account, f := range failures {
|
||||||
|
failing[account] = map[string]any{
|
||||||
|
"count": f.Count,
|
||||||
|
"last-error": f.LastError,
|
||||||
|
"since": f.Since.UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out["failing"] = failing
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,569 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeStore is Store kept in memory, for the test alone.
|
||||||
|
type fakeStore struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
handles map[string]bool
|
||||||
|
statuses map[string]AccountStatus
|
||||||
|
failures map[string]FailureStatus
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFakeStore() *fakeStore {
|
||||||
|
return &fakeStore{
|
||||||
|
handles: map[string]bool{},
|
||||||
|
statuses: map[string]AccountStatus{},
|
||||||
|
failures: map[string]FailureStatus{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) HandleSeen(handle string) (bool, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
return s.handles[handle], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) MarkHandle(handle string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.handles[handle] = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) PutStatus(account string, st AccountStatus) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.statuses[account] = st
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) Statuses() (map[string]AccountStatus, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := map[string]AccountStatus{}
|
||||||
|
for k, v := range s.statuses {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) PutFailure(account string, f FailureStatus) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.failures[account] = f
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) ClearFailure(account string) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
delete(s.failures, account)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *fakeStore) Failures() (map[string]FailureStatus, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
out := map[string]FailureStatus{}
|
||||||
|
for k, v := range s.failures {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeClock is a clock the test moves by hand, so Flush's window needs no real sleep.
|
||||||
|
type fakeClock struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
now time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeClock) Now() time.Time {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return c.now
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *fakeClock) Advance(d time.Duration) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
c.now = c.now.Add(d)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTestNotifier(t *testing.T, clock *fakeClock, store *fakeStore, sent *[]map[string]any) *Notifier {
|
||||||
|
t.Helper()
|
||||||
|
return &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
*sent = append(*sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mail(account string, uid int, messageID, subject, senderName string) EventBody {
|
||||||
|
return EventBody{Account: account, Folder: "INBOX", UID: uint32(uid), MessageID: messageID, Subject: subject,
|
||||||
|
From: []Person{{Name: senderName, Address: "s@example.com"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder is the review's first test for BLOCKING 1: Flush
|
||||||
|
// used to walk a map, so which account's notice went out first was whichever order Go's map iteration gave
|
||||||
|
// that run. Run with -count=50: green every time only once Flush sorts the accounts itself.
|
||||||
|
func TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
for _, account := range []string{"zeta", "mango", "alpha", "delta"} {
|
||||||
|
if err := n.Add(mail(account, 1, account+"-m1", account, "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(%s): %v", account, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"alpha", "delta", "mango", "zeta"}
|
||||||
|
if len(sent) != len(want) {
|
||||||
|
t.Fatalf("sent %d notice(s), want %d: %+v", len(sent), len(want), sent)
|
||||||
|
}
|
||||||
|
for i, account := range want {
|
||||||
|
if sent[i]["summary"] != account {
|
||||||
|
t.Fatalf("sent[%d]'s summary is %v, want %q: accounts must be flushed in sorted order: %+v",
|
||||||
|
i, sent[i]["summary"], account, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts is the review's second test for BLOCKING 2:
|
||||||
|
// a notice node-notifier.send refuses stays pending (nothing is deleted before it is shown), is tried again
|
||||||
|
// at the next Flush no sooner than RetryBackoff, and another account's notice still goes out meanwhile.
|
||||||
|
func TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
failSend := true
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
if args["summary"] == "Broken" && failSend {
|
||||||
|
return errors.New("node-notifier.send: no answer")
|
||||||
|
}
|
||||||
|
sent = append(sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := n.Add(mail("broke", 1, "b1", "Broken", "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(broke): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail("ok", 1, "o1", "Fine", "Someone")); err != nil {
|
||||||
|
t.Fatalf("Add(ok): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 || sent[0]["summary"] != "Fine" {
|
||||||
|
t.Fatalf("sent = %+v, want only ok's notice: broke's failure must not hold it up", sent)
|
||||||
|
}
|
||||||
|
if failures, err := store.Failures(); err != nil || failures["broke"].Count != 1 {
|
||||||
|
t.Fatalf("Store.Failures() = %+v, %v; want broke recorded once", failures, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Before RetryBackoff has passed, broke's notice is not retried yet.
|
||||||
|
clock.Advance(30 * time.Second)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d notice(s) before RetryBackoff elapsed, want 1 (broke not retried yet): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once RetryBackoff has passed, the next Flush tries it again; nothing was lost.
|
||||||
|
failSend = false
|
||||||
|
clock.Advance(RetryBackoff)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 2 || sent[1]["summary"] != "Broken" {
|
||||||
|
t.Fatalf("sent = %+v, want broke's notice too, after RetryBackoff", sent)
|
||||||
|
}
|
||||||
|
if failures, _ := store.Failures(); len(failures) != 0 {
|
||||||
|
t.Errorf("Store.Failures() = %+v, want it cleared once broke's notice was shown", failures)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd is the review's third test (BLOCKING 3,
|
||||||
|
// ADR 0378 §9, the lead's reading, the fold notice's wording updated by review round 3 of issue 800): the
|
||||||
|
// first three of an account's qualifying mail within a minute are shown one by one as Flush runs; the fourth
|
||||||
|
// onward is folded and shown as one notice once the window — a minute from the first of them — is over.
|
||||||
|
func TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 4; i++ {
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
clock.Advance(10 * time.Second)
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("4 mails 10s apart: sent %d notice(s) before the window ends, want 3 shown one by one: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The window is over a minute after it opened (the first mail): the fourth, folded, is shown now.
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("after the window ended, sent %d notice(s), want 4 (the folded fourth mail too): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "1 more new mail in home" || !strings.Contains(sent[3]["body"].(string), "Sender 4") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what the fold holds (1), naming Sender 4", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestARedeliveredEventIsNeverShownOrCountedTwice is the review's fourth test: a redelivery of a mail already
|
||||||
|
// shown is dropped (the Store knows it), and a redelivery of a mail still buffered, waiting to be folded, is
|
||||||
|
// never counted a second time toward the fold.
|
||||||
|
func TestARedeliveredEventIsNeverShownOrCountedTwice(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
one := mail(account, 1, "m1", "Hi", "Alice")
|
||||||
|
if err := n.Add(one); err != nil {
|
||||||
|
t.Fatalf("Add: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 1", len(sent))
|
||||||
|
}
|
||||||
|
if err := n.Add(one); err != nil { // redelivered after it was shown
|
||||||
|
t.Fatalf("Add (redelivered, already shown): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("a redelivered, already-shown event was shown again: sent now %+v", sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 2; i <= 4; i++ {
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), "Hi", fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fourth := mail(account, 4, "m4", "Hi", "Sender 4")
|
||||||
|
if err := n.Add(fourth); err != nil { // redelivered while still buffered, folded, not yet shown
|
||||||
|
t.Fatalf("Add (redelivered, buffered): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 (one, mail 2, mail 3, and one fold notice): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "1 more new mail in home" {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count the redelivered fourth mail once, not twice, among the fold's 1", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange is the review's fifth test: the dedupe key is the
|
||||||
|
// account and the message id wherever one is given, so a UIDVALIDITY reset — which changes every UID in a
|
||||||
|
// folder — cannot make an already-shown mail look new. Only a message with no message id falls back to its
|
||||||
|
// folder, UIDVALIDITY and UID, and that fallback still tells two different mails apart.
|
||||||
|
func TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
first := EventBody{Account: account, Folder: "INBOX", UID: 5, UIDValidity: 1, MessageID: "stable-id",
|
||||||
|
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(first); err != nil {
|
||||||
|
t.Fatalf("Add: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("sent %d, want 1", len(sent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// The same message, redelivered after a UIDVALIDITY reset changed its UID: same message id, not shown again.
|
||||||
|
resent := EventBody{Account: account, Folder: "INBOX", UID: 99, UIDValidity: 2, MessageID: "stable-id",
|
||||||
|
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(resent); err != nil {
|
||||||
|
t.Fatalf("Add (UIDVALIDITY reset, same message id): %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 1 {
|
||||||
|
t.Fatalf("a mail with the same message id after a UIDVALIDITY reset was shown again: sent now %+v", sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two different mails with no message id, at the same UID but a different UIDVALIDITY, are not confused.
|
||||||
|
noID1 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 1, Subject: "No id",
|
||||||
|
From: []Person{{Address: "a@example.com"}}}
|
||||||
|
noID2 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 2, Subject: "No id, new validity",
|
||||||
|
From: []Person{{Address: "a@example.com"}}}
|
||||||
|
if err := n.Add(noID1); err != nil {
|
||||||
|
t.Fatalf("Add noID1: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Add(noID2); err != nil {
|
||||||
|
t.Fatalf("Add noID2: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("sent %d, want 3: noID1 and noID2 are different mails, their UIDVALIDITY differs: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestQualifyingRulesStillHold keeps the original qualifying rules proven (an already-read mail, a mail from
|
||||||
|
// the account's own address, and a mail outside the inbox are never shown), now that buffering is per window
|
||||||
|
// rather than per quiet period.
|
||||||
|
func TestQualifyingRulesStillHold(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error { sent = append(sent, args); return nil },
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{"work": "me@work.example"}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
already := EventBody{Account: "work", Folder: "INBOX", UID: 2, MessageID: "m2", Subject: "Old",
|
||||||
|
From: []Person{{Address: "bob@example.com"}}, Flags: []string{"\\Seen"}}
|
||||||
|
own := EventBody{Account: "work", Folder: "INBOX", UID: 3, MessageID: "m3", Subject: "Sent",
|
||||||
|
From: []Person{{Address: "me@work.example"}}}
|
||||||
|
elsewhere := EventBody{Account: "work", Folder: "Archive", UID: 4, MessageID: "m4", Subject: "Filed",
|
||||||
|
From: []Person{{Address: "carol@example.com"}}}
|
||||||
|
for _, e := range []EventBody{already, own, elsewhere} {
|
||||||
|
if err := n.Add(e); err != nil {
|
||||||
|
t.Fatalf("Add(%+v): %v", e, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 0 {
|
||||||
|
t.Fatalf("sent %+v, want nothing: an already-read mail, the account's own mail and mail outside the inbox never qualify", sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail is review round 2 of issue 800's first test (BLOCKING):
|
||||||
|
// a send that fails keeps its mail pending; once the cooldown has passed, but before the next Flush retries
|
||||||
|
// it, a new qualifying mail arrives. Neither mail is lost: both are shown once the retry succeeds.
|
||||||
|
func TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
failSend := true
|
||||||
|
n := &Notifier{
|
||||||
|
Now: clock.Now,
|
||||||
|
Send: func(args map[string]any) error {
|
||||||
|
if failSend {
|
||||||
|
return errors.New("node-notifier.send: no answer")
|
||||||
|
}
|
||||||
|
sent = append(sent, args)
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
|
||||||
|
Store: store,
|
||||||
|
EmitShown: func(string, string) error { return nil },
|
||||||
|
Logf: t.Logf,
|
||||||
|
}
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
if err := n.Add(mail(account, 1, "m1", "Mail 1", "Sender 1")); err != nil {
|
||||||
|
t.Fatalf("Add mail 1: %v", err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 0 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 0: the send failed", len(sent))
|
||||||
|
}
|
||||||
|
|
||||||
|
// The cooldown passes, but no retry has happened yet (Flush is not called again before the new mail
|
||||||
|
// arrives): a buffer this old must not be thrown away for a fresh one.
|
||||||
|
clock.Advance(RetryBackoff)
|
||||||
|
if err := n.Add(mail(account, 2, "m2", "Mail 2", "Sender 2")); err != nil {
|
||||||
|
t.Fatalf("Add mail 2: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
failSend = false
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 2 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 2 (mail 1's retry and mail 2, neither lost): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota is review round 3 of issue 800's first test,
|
||||||
|
// replacing round 2's (which contradicted round 3's rule): N, for a mail arriving at t, is every qualifying
|
||||||
|
// mail of the account — shown or folded alike — that arrived in the 60 seconds up to and including t. Mails at
|
||||||
|
// 0, 50, 55, 70, 75 and 80 seconds: at 70s the preceding 60 seconds (from 10s) hold 50, 55 and 70 themselves
|
||||||
|
// (0s has aged out), N=3, so 70 is still shown at once, not folded; at 75s the preceding 60 seconds (from 15s)
|
||||||
|
// still hold 50 too, so it, 55, 70 and 75 themselves make N=4, and 75 folds; 80 folds with it (its own
|
||||||
|
// preceding 60 seconds hold 50, 55, 70, 75 and 80, N=5). The fold is due 60 seconds after its own first mail
|
||||||
|
// (75s), at 135s, as one notice naming only what it holds (2) and the latest sender (80s's).
|
||||||
|
func TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
deltas := []time.Duration{0, 50 * time.Second, 5 * time.Second, 15 * time.Second, 5 * time.Second, 5 * time.Second}
|
||||||
|
for i, d := range deltas {
|
||||||
|
clock.Advance(d)
|
||||||
|
if err := n.Add(mail(account, i+1, fmt.Sprintf("m%d", i+1), fmt.Sprintf("Mail %d", i+1), fmt.Sprintf("Sender %d", i+1))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 shown one by one (0s, 50s, 55s, 70s): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3", "Mail 4"} {
|
||||||
|
if sent[i]["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fold's own window — a minute from its first mail, 75s — ends at 135s.
|
||||||
|
clock.Advance(55 * time.Second) // now at 80s + 55s = 135s
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 5 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 5 (75s and 80s folded into one, at 135s): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[4]["summary"] != "2 more new mails in home" || !strings.Contains(sent[4]["body"].(string), "Sender 6") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 6 (80s, the latest)", sent[4])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestASteadyTrickleNeverFoldsForever is review round 3 of issue 800's second test: the bug the round found
|
||||||
|
// (a session that only resets after a quiet minute folds a steady stream forever, even without a burst). One
|
||||||
|
// mail every 50 seconds, 20 of them: each one's preceding 60 seconds holds only the one before it, so N never
|
||||||
|
// exceeds 2, and all 20 are shown at once — none ever fold.
|
||||||
|
func TestASteadyTrickleNeverFoldsForever(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 20; i++ {
|
||||||
|
if i > 1 {
|
||||||
|
clock.Advance(50 * time.Second)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if len(sent) != i {
|
||||||
|
t.Fatalf("after mail %d, sent %d notice(s), want %d: a steady trickle must never fold: %+v", i, len(sent), i, sent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, got := range sent {
|
||||||
|
want := fmt.Sprintf("Mail %d", i+1)
|
||||||
|
if got["summary"] != want {
|
||||||
|
t.Errorf("sent[%d] = %+v, want summary %q", i, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestFiveMailsWithinTenSecondsFoldAfterThree is review round 3 of issue 800's third test: five mails within
|
||||||
|
// ten seconds — the first three shown at once, the fourth opens a fold, the fifth joins it, and the fold is
|
||||||
|
// shown 60 seconds after the fourth (the fold's own first mail) as one notice.
|
||||||
|
func TestFiveMailsWithinTenSecondsFoldAfterThree(t *testing.T) {
|
||||||
|
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
|
||||||
|
store := newFakeStore()
|
||||||
|
var sent []map[string]any
|
||||||
|
n := newTestNotifier(t, clock, store, &sent)
|
||||||
|
|
||||||
|
account := "home"
|
||||||
|
for i := 1; i <= 5; i++ {
|
||||||
|
if i > 1 {
|
||||||
|
clock.Advance(2 * time.Second)
|
||||||
|
}
|
||||||
|
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
|
||||||
|
t.Fatalf("Add mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush at mail %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(sent) != 3 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 3 shown one by one, the fourth and fifth folded, not yet due: %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The fold is due 60 seconds after its own first mail (the fourth, at 6s).
|
||||||
|
clock.Advance(Window)
|
||||||
|
if err := n.Flush(); err != nil {
|
||||||
|
t.Fatalf("Flush: %v", err)
|
||||||
|
}
|
||||||
|
if len(sent) != 4 {
|
||||||
|
t.Fatalf("sent %d notice(s), want 4 (one fold notice carrying the fourth and fifth): %+v", len(sent), sent)
|
||||||
|
}
|
||||||
|
if sent[3]["summary"] != "2 more new mails in home" || !strings.Contains(sent[3]["body"].(string), "Sender 5") {
|
||||||
|
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 5 (the latest)", sent[3])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestRunStartsTheWatcherEvenWhenStdinIsAtEOF proves the watcher runs beside stdio.Serve in the one process the
|
||||||
|
// node's runtime launches (novox/hq ADR 0378 §9, as mesh-knowledge's serve starts go hear(k) and go
|
||||||
|
// background(k) before stdio.Serve), not only in a separate process something keeps asking over stdin. run is
|
||||||
|
// given stdin already at EOF, exactly what systemd gave the old separate `notify` process: it exited at once,
|
||||||
|
// every five seconds (novox/hq issue 859). The watcher must still be running once run has returned — proved by
|
||||||
|
// making it actually answer after run returns, not by the absence of a signal that nothing but the test's own
|
||||||
|
// cleanup could ever raise.
|
||||||
|
func TestRunStartsTheWatcherEvenWhenStdinIsAtEOF(t *testing.T) {
|
||||||
|
started := make(chan struct{})
|
||||||
|
ping := make(chan struct{})
|
||||||
|
pong := make(chan struct{})
|
||||||
|
|
||||||
|
fakeWatcher := func(n *Notifier) {
|
||||||
|
close(started)
|
||||||
|
for range ping {
|
||||||
|
pong <- struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
n := &Notifier{Store: newFakeStore()}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- run(n, fakeWatcher, strings.NewReader(""), io.Discard, nil) }()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher was not started before stdio.Serve returned")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run over stdin at EOF: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("run did not return once stdin reached EOF, as systemd gives a process nobody talks to")
|
||||||
|
}
|
||||||
|
|
||||||
|
// run has returned (stdio.Serve hit EOF at once); the watcher must still be live and answering, not merely
|
||||||
|
// not yet torn down. A watcher that stopped when run returned would never take this ping.
|
||||||
|
select {
|
||||||
|
case ping <- struct{}{}:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher is no longer running once run returned: it must not stop just because stdio.Serve did")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-pong:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher did not answer once run returned")
|
||||||
|
}
|
||||||
|
close(ping)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits proves watch no longer exits the process on a
|
||||||
|
// subscribe the runtime refuses — the old os.Exit(1) took mail-notifier.status and mail_notifier_health down
|
||||||
|
// with it — but retries with backoff and logs each refusal, as modules/mesh-knowledge's hear does (novox/hq
|
||||||
|
// issue 859, review round 1). A subscribe that fails twice then succeeds must leave the process serving (run
|
||||||
|
// still returns normally over stdin at EOF, never os.Exit) and the watcher subscribed (its third, successful
|
||||||
|
// call).
|
||||||
|
func TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits(t *testing.T) {
|
||||||
|
var calls int32
|
||||||
|
subscribed := make(chan struct{})
|
||||||
|
fakeSubscribe := func(pattern string, handle func(stdio.Envelope) error) error {
|
||||||
|
if atomic.AddInt32(&calls, 1) <= 2 {
|
||||||
|
return errors.New("the runtime refused this subscribe")
|
||||||
|
}
|
||||||
|
close(subscribed)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
noSleep := func(time.Duration) {}
|
||||||
|
|
||||||
|
n := &Notifier{Store: newFakeStore()}
|
||||||
|
n.init()
|
||||||
|
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- run(n, func(n *Notifier) { watchWith(n, fakeSubscribe, noSleep) },
|
||||||
|
strings.NewReader(""), io.Discard, nil)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run over stdin at EOF: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("run did not return once stdin reached EOF: a subscribe refusal must not hold up or kill the process")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-subscribed:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("the watcher never subscribed after its first two refusals")
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := atomic.LoadInt32(&calls); got != 3 {
|
||||||
|
t.Fatalf("subscribe was called %d time(s), want 3 (two refusals, then the one that succeeded)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// TestSendKeyNamesThisMachine proves Send asks its own machine's node-notifier, not the bare seat: a bare
|
||||||
|
// "seat:node-notifier.send" is answered by whichever machine's node-notifier takes it first, so notices sent
|
||||||
|
// from g14 and shanks were lost to "503 no responders" or another machine's desktop (novox/hq issue 865).
|
||||||
|
// desk-channel names the machine it asks the same way ("seat:node-notifier.send@"+machine,
|
||||||
|
// modules/desk-channel/cmd/desk-channel/main.go); sendKey must read this machine's own name the same way the
|
||||||
|
// SDK's own Emit does, from MESH_NODE, which the runtime sets.
|
||||||
|
func TestSendKeyNamesThisMachine(t *testing.T) {
|
||||||
|
t.Setenv("MESH_NODE", "g14")
|
||||||
|
|
||||||
|
if got, want := sendKey(), "seat:node-notifier.send@g14"; got != want {
|
||||||
|
t.Fatalf("sendKey() = %q, want %q (it must name this machine, as desk-channel does)", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
module mesh-mail-notifier
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
{
|
||||||
|
"module": "mesh-mail-notifier",
|
||||||
|
"version": "1",
|
||||||
|
"runs-as": "mesh-mail-notifier",
|
||||||
|
"claims": [
|
||||||
|
{
|
||||||
|
"name": "mail-notifier",
|
||||||
|
"scope": "node",
|
||||||
|
"serves": ["status"]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"consumes": [
|
||||||
|
"mesh-mail.mail.*.message.added"
|
||||||
|
],
|
||||||
|
"invokes": [
|
||||||
|
"seat:node-notifier.send",
|
||||||
|
"seat:mail-store.accounts"
|
||||||
|
],
|
||||||
|
"emits": [
|
||||||
|
"shown"
|
||||||
|
],
|
||||||
|
"state": [
|
||||||
|
{
|
||||||
|
"name": "shown",
|
||||||
|
"ttl-seconds": 2592000
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tools": [
|
||||||
|
"mail-notifier.status",
|
||||||
|
"mail_notifier_health"
|
||||||
|
],
|
||||||
|
"own-secrets": {
|
||||||
|
"broker": "${dir:state}/broker"
|
||||||
|
},
|
||||||
|
"secrets-owner": "mesh-mail-notifier",
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "account",
|
||||||
|
"type": "user",
|
||||||
|
"name": "mesh-mail-notifier",
|
||||||
|
"shell": "/usr/bin/nologin",
|
||||||
|
"home": "/var/lib/mesh-mail-notifier"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "state",
|
||||||
|
"type": "directory",
|
||||||
|
"mode": "0700",
|
||||||
|
"owner": "mesh-mail-notifier",
|
||||||
|
"place": "."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "tools",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/mesh-mail-notifier",
|
||||||
|
"binary": "mesh-mail-notifier",
|
||||||
|
"loads": [
|
||||||
|
"mesh-mail-notifier"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
# mesh-mail — the operator's mail accounts, watched and said on the bus
|
||||||
|
|
||||||
|
One module holds the operator's mail accounts, watches them, and says each change of a message on the bus for the
|
||||||
|
modules the operator grants it to (novox/hq ADR 0359, research 049 document 02). One instance, on the control-node.
|
||||||
|
Reading is the first stage: accounts of the kind `imap`, read-only — nothing here flags, moves, deletes or drafts.
|
||||||
|
Sending is `mail_send` alone (novox/hq ADR 0361 §2): it sends nothing itself, and reaches an account's outgoing
|
||||||
|
server only once the operator approves it on a channel that proves who answered.
|
||||||
|
|
||||||
|
## Its seats (novox/hq ADR 0378 as amended by ADR 0379, issue 800 part A)
|
||||||
|
|
||||||
|
It declares three mail seats and claims the one it holds:
|
||||||
|
|
||||||
|
- **mail-store** (held once for the mesh) — serves `accounts`, `search`, `read` and `send`; mesh-mail claims it,
|
||||||
|
through the same handlers as its own `mail_*` tools. The catalogue's own check (`checks/mailstore`) refuses any
|
||||||
|
other manifest that claims it.
|
||||||
|
- **mail-webclient** (held once for the mesh) — serves `open` and `status`; unheld until issue 580.
|
||||||
|
- **mail-notifier** (held once per machine, needing `node-notifier` there) — serves `status`; held by
|
||||||
|
mesh-mail-notifier (issue 800 part B), a module of its own built from this one's source, which tells the
|
||||||
|
operator of each new inbox mail through `node-notifier.send`.
|
||||||
|
|
||||||
|
## What it does
|
||||||
|
|
||||||
|
- **Watches.** For each account it keeps one connection: IDLE on the inbox, ended and begun again every two minutes
|
||||||
|
(which proves the server still answers), and a look at every other folder every five minutes. Folders are opened
|
||||||
|
read-only and bodies are peeked, so watching never marks a message seen.
|
||||||
|
- **Indexes.** One row per message and folder in its own database (`postgres-database`): metadata, flags, the text
|
||||||
|
for search, and the attachments' references. Each attachment is kept once, by its SHA-256, in its bucket
|
||||||
|
(`s3-bucket`, from `minio`).
|
||||||
|
- **Says each change.** A message added, its flags changed, or removed is said as the event
|
||||||
|
`mail.<account>.message.<added|changed|removed>`: account, folder, uid, message id, date, sender, recipients,
|
||||||
|
subject, size, flags and attachment references; never the body, never an attachment's bytes. The events go to the
|
||||||
|
module's own stream `MAIL`, kept seven days, never to the shared EVENTS stream (novox/hq ADR 0363); a module hears an
|
||||||
|
account's part only when its `consumes` names mesh-mail and the setting `grants` names it for that account. The first look at a
|
||||||
|
folder indexes what is there and says nothing: the bus hears what changed since the watch began. The keeper writes
|
||||||
|
each event in the same step as the change it says; the bundle says it through the runtime and removes it once the
|
||||||
|
bus took it.
|
||||||
|
|
||||||
|
## Its settings, both trusted
|
||||||
|
|
||||||
|
`accounts` — per account name (a lower-case token, which names the account on the bus and its secret):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"mesh": {"kind": "imap", "address": "<the address>",
|
||||||
|
"incoming": {"host": "<the IMAP server>", "port": 993, "security": "tls"},
|
||||||
|
"outgoing": {"host": "<the SMTP server>", "port": 465, "security": "tls"},
|
||||||
|
"folders": ["optional: the folders watched besides the inbox; every folder when left out"]}}
|
||||||
|
```
|
||||||
|
|
||||||
|
`outgoing` is the server `mail_send` sends through: host, port, and `security` `tls` (465, implicit) or `starttls`
|
||||||
|
(587). A second `imap` account on another server is one more entry and one more password; no code changes. `user`
|
||||||
|
signs in under another name than the address.
|
||||||
|
|
||||||
|
`grants` — per account, who may read its mail, and who may send from it: a module's name (on every machine),
|
||||||
|
`<machine>.<module>`, or `person.<name>`. Nobody by default. It restricts the stream `MAIL` on the bus (a person's
|
||||||
|
entry grants nothing there) and the tools `mail_search`, `mail_read` and `mail_send` — the same check for all three.
|
||||||
|
|
||||||
|
The password of an account is the own secret `mail-credential-<account>`, given at the desk; the mesh never makes one.
|
||||||
|
|
||||||
|
## Its tools
|
||||||
|
|
||||||
|
| Tool | Answers |
|
||||||
|
|---|---|
|
||||||
|
| `mail_accounts` | each account's kind, state and why, when its server last answered, its folders; the address only to a caller granted it |
|
||||||
|
| `mail_search` | messages of the accounts the caller is granted, by every word of a query, newest first, no body |
|
||||||
|
| `mail_read` | one message's metadata and text, or one attachment's bytes, to a caller granted its account |
|
||||||
|
| `mail_health` | healthy; unhealthy naming the account and why (connection down, or no answer for five minutes); waits naming `mail-credential-<account>` while its password is not given |
|
||||||
|
| `mail_send` | raises an ask on the operator channel (the account, the recipients, the subject, the first lines) for a caller granted the account; answers at once with the ask's id. Sends nothing itself |
|
||||||
|
| `mail_send_status` | where a send stands, by its ask id: waiting, sent, declined, ended without a choice, or not proven |
|
||||||
|
|
||||||
|
The caller is the runtime's word, never an argument. `mail_send` sends through the account's outgoing server only
|
||||||
|
once the operator's Approve comes back on a channel that proves who answered, and once: a warrant delivered twice
|
||||||
|
never sends twice (novox/hq ADR 0361 §2).
|
||||||
|
|
||||||
|
The mail-store seat's four verbs, `mail-store.accounts`, `mail-store.search`, `mail-store.read` and
|
||||||
|
`mail-store.send`, answer through the same handlers as `mail_accounts`, `mail_search`, `mail_read` and `mail_send`
|
||||||
|
above — the same grants, the same refusals, the same ask-then-Approve for a send.
|
||||||
|
|
||||||
|
## Its processes
|
||||||
|
|
||||||
|
`prepare` brings the index's schema to this version's (numbered migrations, and `schema.sql` for a fresh database).
|
||||||
|
`keep` watches the accounts and is judged by its unit. `watch` judges every minute and is judged by `mail_health`.
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// mail_send raises its ask on the operator channel, and messenger's content rule (ADR 0253 §3) refuses any
|
||||||
|
// message that carries a mail address, a URL or a bare host name. ADR 0361 §2 needs the ask to show the
|
||||||
|
// account, the recipients, the subject and the first lines regardless — so the ask must say all of that in
|
||||||
|
// a form the content rule lets through (novox/hq issue 868, "Decided" note and review round 1). These tests
|
||||||
|
// build real asks through mail_send and judge them with the real rule, copied verbatim as content_rule.go.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
)
|
||||||
|
|
||||||
|
// wholeAsk is every word the operator can ever see for one ask: its Headline, Explanation and Details
|
||||||
|
// together. Useful for a readability check (nothing unmasked is visible anywhere), but never for judging
|
||||||
|
// whether messenger would refuse the ask as a whole — judgedWords is that (novox/hq issue 868, review
|
||||||
|
// round 1): Details is filtered line by line, not judged with the rest (holder.go's sayOn).
|
||||||
|
func wholeAsk(r *sendRouter, id string) string {
|
||||||
|
a := r.published[id]
|
||||||
|
return a.Headline + "\n" + a.Explanation + "\n" + a.Details
|
||||||
|
}
|
||||||
|
|
||||||
|
// judgedWords is exactly the text messenger refuses (or lets through) an ask as a whole on (asks.go's
|
||||||
|
// TakeAsk): its headline, its explanation, what happens at expiry, and each option's label and effect.
|
||||||
|
// Details is never part of it.
|
||||||
|
func judgedWords(a asks.Ask) string {
|
||||||
|
words := []string{a.Headline, a.Explanation, a.OnExpiry}
|
||||||
|
for _, o := range a.Options {
|
||||||
|
words = append(words, o.Label, o.Does)
|
||||||
|
}
|
||||||
|
return strings.Join(words, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// mustNotBeRefused fails the test naming what the content rule refused, and why — judged the way messenger
|
||||||
|
// itself judges an ask as a whole, on judgedWords alone.
|
||||||
|
func mustNotBeRefused(t *testing.T, a asks.Ask) {
|
||||||
|
t.Helper()
|
||||||
|
if refusal, ok := Check(judgedWords(a)); !ok {
|
||||||
|
t.Fatalf("the content rule refuses the ask: %s\n%s", refusal, judgedWords(a))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (1) A recipient that is one of the operator's own mesh-mail accounts (sendTestBundle's "mesh", address
|
||||||
|
// jschoubben@novox.be) is named by the account's name alone: the ask carries no mail address at all, and
|
||||||
|
// passes the real content rule. This is the first test issue 868's decision asked for; it fails today
|
||||||
|
// because AskMailSend puts the raw address into the ask.
|
||||||
|
func TestMailSendAskForOwnAccountRecipientCarriesNoAddress(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
|
||||||
|
args := map[string]any{"account": "mesh", "to": []any{"jschoubben@novox.be"},
|
||||||
|
"subject": "Testing issue 868", "body": "Does the ask reach the operator now?"}
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", args)
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
if json.Unmarshal([]byte(text), &out) != nil {
|
||||||
|
t.Fatalf("not JSON: %s", text)
|
||||||
|
}
|
||||||
|
id, _ := out["ask"].(string)
|
||||||
|
if id == "" {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
whole := wholeAsk(router, id)
|
||||||
|
if strings.Contains(whole, "@") {
|
||||||
|
t.Errorf("the ask carries a mail address for its own account recipient:\n%s", whole)
|
||||||
|
}
|
||||||
|
if !strings.Contains(whole, "your account mesh") {
|
||||||
|
t.Errorf("the ask does not name the own account:\n%s", whole)
|
||||||
|
}
|
||||||
|
mustNotBeRefused(t, router.published[id])
|
||||||
|
}
|
||||||
|
|
||||||
|
// (2) A recipient outside the operator's own accounts is said without its address or host: its first
|
||||||
|
// letter, an ellipsis and the domain's labels as words.
|
||||||
|
func TestMailSendAskForOutsideRecipientCarriesNoAddress(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", sendArgs) // to: ops@example.org, lead@example.org
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(text), &out)
|
||||||
|
id := out["ask"].(string)
|
||||||
|
whole := wholeAsk(router, id)
|
||||||
|
if strings.Contains(whole, "@") || strings.Contains(whole, "example.org") {
|
||||||
|
t.Errorf("the ask carries the recipients' address or host:\n%s", whole)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"o… at example org", "l… at example org"} {
|
||||||
|
if !strings.Contains(whole, want) {
|
||||||
|
t.Errorf("the ask does not say %q:\n%s", want, whole)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mustNotBeRefused(t, router.published[id])
|
||||||
|
}
|
||||||
|
|
||||||
|
// (3) More than three recipients are counted rather than said one by one.
|
||||||
|
func TestMailSendAskForManyRecipientsCountsTheRest(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
|
||||||
|
args := map[string]any{"account": "mesh",
|
||||||
|
"to": []any{"a@example.org", "b@example.org", "c@example.org", "d@example.org", "e@example.org"},
|
||||||
|
"subject": "Many recipients", "body": "body"}
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", args)
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(text), &out)
|
||||||
|
id := out["ask"].(string)
|
||||||
|
whole := wholeAsk(router, id)
|
||||||
|
if strings.Contains(whole, "@") {
|
||||||
|
t.Errorf("the ask carries a mail address among many recipients:\n%s", whole)
|
||||||
|
}
|
||||||
|
if !strings.Contains(whole, "and 2 more") {
|
||||||
|
t.Errorf("the ask does not count the recipients past the first three:\n%s", whole)
|
||||||
|
}
|
||||||
|
mustNotBeRefused(t, router.published[id])
|
||||||
|
}
|
||||||
|
|
||||||
|
// (4) A subject holding a mail address and a link has both masked before the ask carries it.
|
||||||
|
func TestMailSendAskMasksAnAddressAndALinkInTheSubject(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
|
||||||
|
args := map[string]any{"account": "mesh", "to": []any{"ops@example.org"},
|
||||||
|
"subject": "Contact ops@example.org or see https://example.com/issue/42",
|
||||||
|
"body": "See https://example.com/issue/42 and write to ops@example.org for details."}
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", args)
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(text), &out)
|
||||||
|
id := out["ask"].(string)
|
||||||
|
whole := wholeAsk(router, id)
|
||||||
|
if strings.Contains(whole, "@") || strings.Contains(whole, "example.com") || strings.Contains(whole, "://") {
|
||||||
|
t.Errorf("the ask carries an unmasked address or link:\n%s", whole)
|
||||||
|
}
|
||||||
|
if !strings.Contains(whole, "[address]") || !strings.Contains(whole, "[link]") {
|
||||||
|
t.Errorf("the ask does not say [address] and [link] in place of what it masked:\n%s", whole)
|
||||||
|
}
|
||||||
|
mustNotBeRefused(t, router.published[id])
|
||||||
|
}
|
||||||
|
|
||||||
|
// (5) review round 1: a subject the masking does not catch (an IP, a two-slash path-shaped reference, a
|
||||||
|
// 40-character hex hash) does not refuse the whole ask: the real content rule (content_rule.go) still judges
|
||||||
|
// the masked explanation as refused, so the subject and first lines are left out of it and pointed at Details
|
||||||
|
// instead — the ask still names the account and recipients, and still reaches the operator.
|
||||||
|
func TestMailSendAskFallsBackToDetailsWhenTheSubjectStillRefuses(t *testing.T) {
|
||||||
|
for _, subject := range []string{
|
||||||
|
"Invoice 2026/10/15",
|
||||||
|
"server 10.0.0.1",
|
||||||
|
"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", // 40 hex characters: reHex's shape
|
||||||
|
} {
|
||||||
|
t.Run(subject, func(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
args := map[string]any{"account": "mesh", "to": []any{"ops@example.org"}, "subject": subject, "body": "body"}
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", args)
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(text), &out)
|
||||||
|
id := out["ask"].(string)
|
||||||
|
a := router.published[id]
|
||||||
|
mustNotBeRefused(t, a)
|
||||||
|
if !strings.Contains(a.Explanation, "mesh") {
|
||||||
|
t.Errorf("the explanation does not name the account:\n%s", a.Explanation)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Explanation, "o… at example org") {
|
||||||
|
t.Errorf("the explanation does not name the recipient:\n%s", a.Explanation)
|
||||||
|
}
|
||||||
|
if strings.Contains(a.Explanation, subject) {
|
||||||
|
t.Errorf("the explanation still carries the subject that refused the ask:\n%s", a.Explanation)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Details, "Details") && !strings.Contains(a.Explanation, "Details") {
|
||||||
|
t.Errorf("the explanation does not point at Details:\n%s", a.Explanation)
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Details, subject) {
|
||||||
|
t.Errorf("Details does not carry the subject (filtered line by line by messenger, as the subject's own line): %s", a.Details)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// (6) review round 1: a recipient written with a display name ("Name <addr>") is matched against the
|
||||||
|
// operator's own accounts, and masked, by its address alone.
|
||||||
|
func TestMailSendAskMatchesADisplayNameRecipientByItsAddress(t *testing.T) {
|
||||||
|
b, router, _ := sendTestBundle(t, Server{Host: "127.0.0.1", Port: 25, Security: "tls"})
|
||||||
|
rt := startBundle(t, b.Tools())
|
||||||
|
|
||||||
|
args := map[string]any{"account": "mesh", "to": []any{"Jochen Schoubben <jschoubben@novox.be>"},
|
||||||
|
"subject": "Testing a display name", "body": "body"}
|
||||||
|
text, isErr := rt.call(SendTool, "novox.triage", args)
|
||||||
|
if isErr {
|
||||||
|
t.Fatalf("%s", text)
|
||||||
|
}
|
||||||
|
var out map[string]any
|
||||||
|
_ = json.Unmarshal([]byte(text), &out)
|
||||||
|
id := out["ask"].(string)
|
||||||
|
whole := wholeAsk(router, id)
|
||||||
|
if strings.Contains(whole, "@") || strings.Contains(whole, "Jochen Schoubben") {
|
||||||
|
t.Errorf("the ask carries the display-name recipient's address or name:\n%s", whole)
|
||||||
|
}
|
||||||
|
if !strings.Contains(whole, "your account mesh") {
|
||||||
|
t.Errorf("the ask does not match the display-name recipient to its own account:\n%s", whole)
|
||||||
|
}
|
||||||
|
mustNotBeRefused(t, router.published[id])
|
||||||
|
}
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The operator's mail accounts and who may read them (novox/hq ADR 0359 §1, §4, §5): two trusted settings, each a
|
||||||
|
// file the mesh writes from the module's settings and changes only on the operator's approval. `accounts` says which
|
||||||
|
// server each account is on and how to sign in (never a credential); `grants` says, per account, which callers may
|
||||||
|
// read its mail. The password of an account is the module's own secret mail-credential-<account>, given at the desk
|
||||||
|
// (ADR 0283 §6), at the path the secrets directory and that name make.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The kinds of account an adapter exists for. Only `imap` is built (ADR 0359 §5); `imap-oauth` and `graph` are later
|
||||||
|
// decisions (§7), and an account of a kind not listed here is refused by name.
|
||||||
|
const KindIMAP = "imap"
|
||||||
|
|
||||||
|
// Kinds is every kind this version has an adapter for.
|
||||||
|
var Kinds = []string{KindIMAP}
|
||||||
|
|
||||||
|
// SecretFamily is the prefix of every account's own secret: mail-credential-<account> (ADR 0283 §6).
|
||||||
|
const SecretFamily = "mail-credential-"
|
||||||
|
|
||||||
|
// accountName is what an account may be called: a member name of the secret family (ADR 0283 §6) and one token of a
|
||||||
|
// bus subject, so never a dot, a slash or an upper-case letter.
|
||||||
|
var accountName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,39}$`)
|
||||||
|
|
||||||
|
// Server is where an account's mail is reached.
|
||||||
|
type Server struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
Port int `json:"port"`
|
||||||
|
// Security is tls (TLS from the first byte) or starttls (plain, then upgraded before signing in). Nothing
|
||||||
|
// signs in over a connection that is not encrypted.
|
||||||
|
Security string `json:"security"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Address is host:port.
|
||||||
|
func (s Server) Address() string { return fmt.Sprintf("%s:%d", s.Host, s.Port) }
|
||||||
|
|
||||||
|
// Account is one of the operator's mail accounts, as the setting `accounts` declares it.
|
||||||
|
type Account struct {
|
||||||
|
// Name is the key the account has in the setting; set when the setting is read.
|
||||||
|
Name string `json:"-"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
// Address is the account's mail address, said only to a caller granted the account.
|
||||||
|
Address string `json:"address"`
|
||||||
|
// User is the name it signs in with; the address when empty.
|
||||||
|
User string `json:"user,omitempty"`
|
||||||
|
Incoming Server `json:"incoming"`
|
||||||
|
Outgoing *Server `json:"outgoing,omitempty"`
|
||||||
|
// Folders are the folders watched besides the inbox; every folder the server lists when empty.
|
||||||
|
Folders []string `json:"folders,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SignInAs is the name the account signs in with.
|
||||||
|
func (a Account) SignInAs() string {
|
||||||
|
if a.User != "" {
|
||||||
|
return a.User
|
||||||
|
}
|
||||||
|
return a.Address
|
||||||
|
}
|
||||||
|
|
||||||
|
// Secret is the name of the account's own secret.
|
||||||
|
func (a Account) Secret() string { return SecretFamily + a.Name }
|
||||||
|
|
||||||
|
// AccountsFile is the shape of the trusted setting `accounts`, as the mesh writes it.
|
||||||
|
type AccountsFile struct {
|
||||||
|
Accounts map[string]Account `json:"accounts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadAccounts reads the accounts setting, refusing it whole when any account in it cannot be watched: an account
|
||||||
|
// half-understood would be watched wrongly, and the operator approved the setting as one.
|
||||||
|
func ReadAccounts(path string) ([]Account, error) {
|
||||||
|
if path == "" {
|
||||||
|
return nil, errors.New("the module was started without its accounts setting (MESH_MAIL_ACCOUNTS)")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the accounts setting cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
return ParseAccounts(raw)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseAccounts reads the accounts setting's content.
|
||||||
|
func ParseAccounts(raw []byte) ([]Account, error) {
|
||||||
|
var f AccountsFile
|
||||||
|
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&f); err != nil {
|
||||||
|
return nil, fmt.Errorf("the accounts setting is not the module's: %v", err)
|
||||||
|
}
|
||||||
|
var out []Account
|
||||||
|
var problems []string
|
||||||
|
for name, a := range f.Accounts {
|
||||||
|
a.Name = name
|
||||||
|
if err := a.Check(); err != nil {
|
||||||
|
problems = append(problems, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
if len(problems) > 0 {
|
||||||
|
sort.Strings(problems)
|
||||||
|
return nil, errors.New("the accounts setting is refused: " + strings.Join(problems, "; "))
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check says why an account cannot be watched, or nil.
|
||||||
|
func (a Account) Check() error {
|
||||||
|
say := func(format string, args ...any) error {
|
||||||
|
return fmt.Errorf("the account %q: %s", a.Name, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
if !accountName.MatchString(a.Name) {
|
||||||
|
return say("is not an account's name: a lower-case letter or digit, then lower-case letters, digits and hyphens, at most 40")
|
||||||
|
}
|
||||||
|
known := false
|
||||||
|
for _, k := range Kinds {
|
||||||
|
known = known || a.Kind == k
|
||||||
|
}
|
||||||
|
if !known {
|
||||||
|
return say("is of the kind %q, and this version watches only %s", a.Kind, strings.Join(Kinds, ", "))
|
||||||
|
}
|
||||||
|
if !strings.Contains(a.Address, "@") {
|
||||||
|
return say("has no mail address")
|
||||||
|
}
|
||||||
|
if err := a.Incoming.check("incoming"); err != nil {
|
||||||
|
return say("%v", err)
|
||||||
|
}
|
||||||
|
if a.Outgoing != nil {
|
||||||
|
if err := a.Outgoing.check("outgoing"); err != nil {
|
||||||
|
return say("%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, f := range a.Folders {
|
||||||
|
if strings.TrimSpace(f) == "" || strings.EqualFold(f, "INBOX") {
|
||||||
|
return say("lists the folder %q, and the inbox is always watched", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Server) check(which string) error {
|
||||||
|
switch {
|
||||||
|
case strings.TrimSpace(s.Host) == "" || strings.ContainsAny(s.Host, " /:"):
|
||||||
|
return fmt.Errorf("its %s server has no host name", which)
|
||||||
|
case s.Port <= 0 || s.Port > 65535:
|
||||||
|
return fmt.Errorf("its %s server's port %d is not a port", which, s.Port)
|
||||||
|
case s.Security != "tls" && s.Security != "starttls":
|
||||||
|
return fmt.Errorf("its %s server's security is %q, and only tls or starttls signs in", which, s.Security)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SecretPath is where the account's own secret is placed in the secrets directory, as the manifest declares the
|
||||||
|
// family: <dir>/mail-credential-<account>.secret.
|
||||||
|
func SecretPath(dir string, a Account) string { return filepath.Join(dir, a.Secret()+".secret") }
|
||||||
|
|
||||||
|
// ReadSecret is the account's password, or given false when nobody gave it yet: the mesh never makes a member of the
|
||||||
|
// family, so a member not given is no file (ADR 0283 §6).
|
||||||
|
func ReadSecret(dir string, a Account) (string, bool, error) {
|
||||||
|
raw, err := os.ReadFile(SecretPath(dir, a))
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", false, err
|
||||||
|
}
|
||||||
|
pw := strings.TrimRight(string(raw), "\r\n")
|
||||||
|
if pw == "" {
|
||||||
|
return "", false, nil
|
||||||
|
}
|
||||||
|
return pw, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SecretGiven says whether the account's own secret was given.
|
||||||
|
//
|
||||||
|
// It looks at the file and never reads it: whoever judges whether a password was given has no need of the password.
|
||||||
|
// A file that is there and not empty was given; whether it signs in is the keeper's to say.
|
||||||
|
func SecretGiven(dir string, a Account) bool {
|
||||||
|
info, err := os.Stat(SecretPath(dir, a))
|
||||||
|
return err == nil && info.Mode().IsRegular() && info.Size() > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- grants ------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Grants is the trusted setting `grants`: per account, the callers that may read its mail (ADR 0359 §4). An entry is a
|
||||||
|
// module's name (granted wherever it runs), a module on one machine (`<machine>.<module>`) or a person
|
||||||
|
// (`person.<name>`), as the runtime names callers.
|
||||||
|
type Grants map[string][]string
|
||||||
|
|
||||||
|
// GrantsFile is the shape of the setting as the mesh writes it.
|
||||||
|
type GrantsFile struct {
|
||||||
|
Grants Grants `json:"grants"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadGrants reads the grants setting. A setting that cannot be read grants nothing, and says so.
|
||||||
|
func ReadGrants(path string) (Grants, error) {
|
||||||
|
if path == "" {
|
||||||
|
return Grants{}, errors.New("the module was started without its grants setting (MESH_MAIL_GRANTS)")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return Grants{}, fmt.Errorf("the grants setting cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var f GrantsFile
|
||||||
|
dec := json.NewDecoder(strings.NewReader(string(raw)))
|
||||||
|
dec.DisallowUnknownFields()
|
||||||
|
if err := dec.Decode(&f); err != nil {
|
||||||
|
return Grants{}, fmt.Errorf("the grants setting is not the module's: %v", err)
|
||||||
|
}
|
||||||
|
if f.Grants == nil {
|
||||||
|
f.Grants = Grants{}
|
||||||
|
}
|
||||||
|
return f.Grants, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unproven is the caller the runtime proved nobody to be; it is granted nothing.
|
||||||
|
const Unproven = "unproven"
|
||||||
|
|
||||||
|
// Allows says whether the caller may read the account's mail. Nobody is granted an account by default, an unproven
|
||||||
|
// caller never, and a wildcard is no entry.
|
||||||
|
func (g Grants) Allows(caller, account string) bool {
|
||||||
|
if caller == "" || caller == Unproven {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, entry := range g[account] {
|
||||||
|
if entry == "" || strings.ContainsAny(entry, "*>") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if entry == caller {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// A module's name alone grants it on every machine: the caller is <machine>.<module>.
|
||||||
|
if !strings.Contains(entry, ".") {
|
||||||
|
if _, module, ok := strings.Cut(caller, "."); ok && module == entry && !strings.HasPrefix(caller, "person.") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Granted is the accounts, of those given, the caller may read.
|
||||||
|
func (g Grants) Granted(caller string, accounts []Account) []string {
|
||||||
|
var out []string
|
||||||
|
for _, a := range accounts {
|
||||||
|
if g.Allows(caller, a.Name) {
|
||||||
|
out = append(out, a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The first stage's account, as the operator sets the setting `accounts` (ADR 0359 §5): its address and its server,
|
||||||
|
// never a credential; its password is the own secret mail-credential-mesh.
|
||||||
|
const stageOne = `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
|
||||||
|
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}}}}`
|
||||||
|
|
||||||
|
func TestTheFirstStagesAccountIsReadAndItsSecretNamed(t *testing.T) {
|
||||||
|
got, err := ParseAccounts([]byte(stageOne))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 || got[0].Name != "mesh" || got[0].Kind != KindIMAP || got[0].SignInAs() != "jschoubben@novox.be" ||
|
||||||
|
got[0].Incoming.Address() != "mail.novox.be:993" || got[0].Secret() != "mail-credential-mesh" || got[0].Outgoing != nil {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
if p := SecretPath("/state", got[0]); p != "/state/mail-credential-mesh.secret" {
|
||||||
|
t.Fatalf("the secret is placed at %s", p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A second account of the kind imap on another server — Gmail with an app password, another member of the family —
|
||||||
|
// is configuration only: no code changes for it. Each account may also say its outgoing server, which mail_send
|
||||||
|
// uses to send through it (novox/hq ADR 0361 §1).
|
||||||
|
func TestASecondIMAPAccountIsConfigurationOnly(t *testing.T) {
|
||||||
|
got, err := ParseAccounts([]byte(`{"accounts": {
|
||||||
|
"mesh": {"kind": "imap", "address": "jschoubben@novox.be", "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}},
|
||||||
|
"gmail": {"kind": "imap", "address": "someone@gmail.com", "incoming": {"host": "imap.gmail.com", "port": 993, "security": "tls"},
|
||||||
|
"outgoing": {"host": "smtp.gmail.com", "port": 465, "security": "tls"}, "folders": ["[Gmail]/Sent Mail"]}}}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(got) != 2 || got[0].Name != "gmail" || got[0].Secret() != "mail-credential-gmail" || got[0].Outgoing.Address() != "smtp.gmail.com:465" ||
|
||||||
|
!reflect.DeepEqual(got[0].Folders, []string{"[Gmail]/Sent Mail"}) {
|
||||||
|
t.Fatalf("%+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAccountThatCannotBeWatchedRefusesTheSetting(t *testing.T) {
|
||||||
|
for name, raw := range map[string]string{
|
||||||
|
"a later kind": `{"accounts": {"work": {"kind": "graph", "address": "a@b.c", "incoming": {"host": "h", "port": 1, "security": "tls"}}}}`,
|
||||||
|
"no encryption": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 143, "security": "none"}}}}`,
|
||||||
|
"a name with a dot": `{"accounts": {"my.box": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"a password in it": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "password": "p", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"no address": `{"accounts": {"x": {"kind": "imap", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"the inbox listed": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "folders": ["INBOX"], "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
|
||||||
|
"a bad outgoing one": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}, "outgoing": {"host": "", "port": 465, "security": "tls"}}}}`,
|
||||||
|
} {
|
||||||
|
if _, err := ParseAccounts([]byte(raw)); err == nil {
|
||||||
|
t.Errorf("%s: accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got, err := ParseAccounts([]byte(`{"accounts": {}}`)); err != nil || len(got) != 0 {
|
||||||
|
t.Fatalf("the manifest's default: %v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Grants (ADR 0359 §4): nobody by default; a module's name grants it wherever it runs; a module on one machine, or a
|
||||||
|
// person, by its full name; never the unproven, never by a wildcard, never one account for another.
|
||||||
|
func TestGrantsAreByAccountAndByProvenCaller(t *testing.T) {
|
||||||
|
g := Grants{"mesh": {"triage", "novox.desk", "person.jochen", "*"}}
|
||||||
|
for caller, want := range map[string]bool{
|
||||||
|
"novox.triage": true, "shanks.triage": true, "novox.desk": true, "g14.desk": false, "person.jochen": true,
|
||||||
|
"person.triage": false, "novox.other": false, Unproven: false, "": false, "novox.*": false,
|
||||||
|
} {
|
||||||
|
if got := g.Allows(caller, "mesh"); got != want {
|
||||||
|
t.Errorf("%s: %v", caller, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if g.Allows("novox.triage", "gmail") || (Grants{}).Allows("novox.triage", "mesh") {
|
||||||
|
t.Fatal("a grant reached an account it does not name")
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "grants.json")
|
||||||
|
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": ["triage"]}}`), 0o600)
|
||||||
|
read, err := ReadGrants(path)
|
||||||
|
if err != nil || !read.Allows("novox.triage", "mesh") {
|
||||||
|
t.Fatalf("%v %v", read, err)
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": "triage"}}`), 0o600)
|
||||||
|
if read, err := ReadGrants(path); err == nil || len(read) != 0 {
|
||||||
|
t.Fatal("a grants setting it cannot read granted something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether a password was given is read from the file's presence and size alone (issue 733): an empty file is not
|
||||||
|
// given, a file the judge cannot read is still given, and the password is never read to say so.
|
||||||
|
func TestASecretIsGivenByItsFileNotByReadingIt(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := testAccount()
|
||||||
|
if SecretGiven(dir, a) {
|
||||||
|
t.Fatal("given before anything was written")
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(SecretPath(dir, a), nil, 0o600)
|
||||||
|
if SecretGiven(dir, a) {
|
||||||
|
t.Fatal("an empty file is a password")
|
||||||
|
}
|
||||||
|
_ = os.WriteFile(SecretPath(dir, a), []byte("pw\n"), 0o600)
|
||||||
|
if pw, given, _ := ReadSecret(dir, a); !given || pw != "pw" || strings.Contains(pw, "\n") {
|
||||||
|
t.Fatalf("%q %v", pw, given)
|
||||||
|
}
|
||||||
|
// Unreadable to the judge, as a secret owned by root is to the watch: still given.
|
||||||
|
if err := os.Chmod(SecretPath(dir, a), 0); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !SecretGiven(dir, a) {
|
||||||
|
t.Fatal("a password the judge cannot read is said not given")
|
||||||
|
}
|
||||||
|
src, _ := os.ReadFile("config.go")
|
||||||
|
body := string(src)
|
||||||
|
body = body[strings.Index(body, "func SecretGiven("):]
|
||||||
|
body = body[:strings.Index(body, "\n}\n")]
|
||||||
|
if strings.Contains(body, "ReadSecret") || strings.Contains(body, "ReadFile") || strings.Contains(body, "os.Open") {
|
||||||
|
t.Fatalf("SecretGiven reads the password:\n%s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// What may leave the mesh (novox/hq to-be 45 §5, research 028 Q8, ADR 0227): roles and words. A
|
||||||
|
// message carrying an address, a path or anything shaped like a secret is refused here, by the holder,
|
||||||
|
// because Telegram is not end-to-end encrypted for bots and this rule is the only thing between a
|
||||||
|
// condition's words and someone else's server. It is not trusted to each source.
|
||||||
|
//
|
||||||
|
// Deliberately wider than it must be: a commit id or a long random name is refused too. A source
|
||||||
|
// that wants its message through says it in words; a refusal is said, never silent.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"unicode"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Refusal says why a text may not leave: the class of what it carried, never the text itself.
|
||||||
|
type Refusal struct {
|
||||||
|
Class string // address, path or secret
|
||||||
|
What string // a few words: "an IPv4 address", "a URL", …
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r Refusal) String() string { return r.Class + " (" + r.What + ")" }
|
||||||
|
|
||||||
|
var (
|
||||||
|
reURL = regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://`)
|
||||||
|
reEmail = regexp.MustCompile(`[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}`)
|
||||||
|
reIPv4 = regexp.MustCompile(`\b\d{1,3}(\.\d{1,3}){3}\b`)
|
||||||
|
reIPv6 = regexp.MustCompile(`(?i)(^|[^0-9a-z:])(([0-9a-f]{1,4}:){4,7}[0-9a-f]{1,4}|([0-9a-f]{1,4}:)*[0-9a-f]{0,4}::([0-9a-f]{1,4}:)*[0-9a-f]{0,4})([^0-9a-z:]|$)`)
|
||||||
|
reMAC = regexp.MustCompile(`(?i)\b([0-9a-f]{2}[:-]){5}[0-9a-f]{2}\b`)
|
||||||
|
rePEM = regexp.MustCompile(`-----BEGIN [A-Z ]+-----`)
|
||||||
|
reJWT = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}`)
|
||||||
|
reBotToken = regexp.MustCompile(`\b\d{6,}:[A-Za-z0-9_-]{30,}`)
|
||||||
|
reKnown = regexp.MustCompile(`\b(gh[pousr]_[A-Za-z0-9]{20,}|glpat-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9_-]{16,}|xox[abprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16})`)
|
||||||
|
reAssigned = regexp.MustCompile(`(?i)\b(password|passwd|passphrase|secret|token|api[_-]?key|apikey|credential|private[_-]?key)\s*[=:]\s*\S`)
|
||||||
|
reHex = regexp.MustCompile(`(?i)\b[0-9a-f]{32,}\b`)
|
||||||
|
reRun = regexp.MustCompile(`[A-Za-z0-9+/=_]{20,}`)
|
||||||
|
reWinPath = regexp.MustCompile(`(?i)\b[a-z]:\\`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// topLevel are names that end a host name — the generic and country ones a mesh's names use, and the
|
||||||
|
// private ones (.internal, .lan, .home, .local) a mesh is likelier to.
|
||||||
|
var topLevel = map[string]bool{}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
for _, t := range strings.Fields(`com net org edu gov mil int io dev app cloud ai co me info biz xyz
|
||||||
|
site online tech page link
|
||||||
|
be nl de fr uk lu eu ch at it es pt se no dk fi pl cz us ca au nz jp cn ru in br ie
|
||||||
|
internal lan home local localdomain corp intranet private arpa test example invalid localhost`) {
|
||||||
|
topLevel[t] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check says whether a text may leave the mesh, and when not, why.
|
||||||
|
func Check(text string) (Refusal, bool) {
|
||||||
|
switch {
|
||||||
|
case reURL.MatchString(text):
|
||||||
|
return Refusal{"address", "a URL"}, false
|
||||||
|
case reEmail.MatchString(text):
|
||||||
|
return Refusal{"address", "a mail address"}, false
|
||||||
|
case reIPv4.MatchString(text):
|
||||||
|
return Refusal{"address", "an IPv4 address"}, false
|
||||||
|
case reMAC.MatchString(text):
|
||||||
|
return Refusal{"address", "a hardware address"}, false
|
||||||
|
case reIPv6.MatchString(text):
|
||||||
|
return Refusal{"address", "an IPv6 address"}, false
|
||||||
|
}
|
||||||
|
if refusal, ok := secretShape(text); !ok {
|
||||||
|
return refusal, false
|
||||||
|
}
|
||||||
|
if reWinPath.MatchString(text) {
|
||||||
|
return Refusal{"path", "a drive path"}, false
|
||||||
|
}
|
||||||
|
if refusal, ok := randomRun(text); !ok {
|
||||||
|
return refusal, false
|
||||||
|
}
|
||||||
|
for _, word := range strings.FieldsFunc(text, func(r rune) bool {
|
||||||
|
return unicode.IsSpace(r) || strings.ContainsRune("\"'`()[]{}<>,;|", r)
|
||||||
|
}) {
|
||||||
|
w := strings.TrimRight(word, ".:!?")
|
||||||
|
if isPath(w) {
|
||||||
|
return Refusal{"path", "a file path"}, false
|
||||||
|
}
|
||||||
|
if isHostName(w) {
|
||||||
|
return Refusal{"address", "a host name"}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Refusal{}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckSecret says whether a text carries a secret's shape — the one class that leaves the mesh nowhere. It is
|
||||||
|
// what an ask's whole words (asks.Ask.Whole, novox/hq issue 383) are held to: on a kind that proves who answers,
|
||||||
|
// a path or an address is what the operator approves and may be shown, a secret never. A path is read as one:
|
||||||
|
// a run with a slash in it is judged piece by piece between the slashes, so "/mnt/Photos_2024/Jochen" is a
|
||||||
|
// path and not a random string (the review of 2026-10-10); the named shapes — a key block, a signed token, a
|
||||||
|
// bot token, a known token shape, a value given to a secret's name, a long hexadecimal string — hold whatever
|
||||||
|
// the run holds.
|
||||||
|
func CheckSecret(text string) (Refusal, bool) {
|
||||||
|
if refusal, ok := secretShape(text); !ok {
|
||||||
|
return refusal, false
|
||||||
|
}
|
||||||
|
return randomRunOutsidePaths(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
// randomRunOutsidePaths is randomRun with each run that holds a slash judged by its pieces between the slashes.
|
||||||
|
func randomRunOutsidePaths(text string) (Refusal, bool) {
|
||||||
|
for _, run := range reRun.FindAllString(text, -1) {
|
||||||
|
for _, piece := range strings.Split(run, "/") {
|
||||||
|
if len(piece) >= 20 && looksRandom(piece) {
|
||||||
|
return Refusal{"secret", "a long random-looking string"}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Refusal{}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretShape is the secret shapes a pattern names.
|
||||||
|
func secretShape(text string) (Refusal, bool) {
|
||||||
|
switch {
|
||||||
|
case rePEM.MatchString(text):
|
||||||
|
return Refusal{"secret", "a key block"}, false
|
||||||
|
case reJWT.MatchString(text):
|
||||||
|
return Refusal{"secret", "a signed token"}, false
|
||||||
|
case reBotToken.MatchString(text):
|
||||||
|
return Refusal{"secret", "a bot token"}, false
|
||||||
|
case reKnown.MatchString(text):
|
||||||
|
return Refusal{"secret", "a known token shape"}, false
|
||||||
|
case reAssigned.MatchString(text):
|
||||||
|
return Refusal{"secret", "a value given to a secret's name"}, false
|
||||||
|
case reHex.MatchString(text):
|
||||||
|
return Refusal{"secret", "a long hexadecimal string"}, false
|
||||||
|
}
|
||||||
|
return Refusal{}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// randomRun is a long unbroken run of characters spread as a random string's are.
|
||||||
|
func randomRun(text string) (Refusal, bool) {
|
||||||
|
for _, run := range reRun.FindAllString(text, -1) {
|
||||||
|
if looksRandom(run) {
|
||||||
|
return Refusal{"secret", "a long random-looking string"}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Refusal{}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isPath: absolute, home-relative or dot-relative, or two separators deep. A mesh address names one
|
||||||
|
// machine and one tool (`ace/postgres.query`) and has one; a ratio ("3/4") has digits only.
|
||||||
|
func isPath(w string) bool {
|
||||||
|
if w == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(w, "/") && len(w) > 1 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, p := range []string{"~/", "./", "../", "$HOME", "${"} {
|
||||||
|
if strings.HasPrefix(w, p) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Count(w, "/") >= 2 || strings.Contains(w, "\\")
|
||||||
|
}
|
||||||
|
|
||||||
|
// isHostName: two names or more, the last a top-level one. A condition key's last name is its kind
|
||||||
|
// (`machine.ace.silent`), which none of these is.
|
||||||
|
func isHostName(w string) bool {
|
||||||
|
w = strings.ToLower(w)
|
||||||
|
if w == "localhost" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
parts := strings.Split(w, ".")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range parts {
|
||||||
|
if p == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return topLevel[parts[len(parts)-1]]
|
||||||
|
}
|
||||||
|
|
||||||
|
// looksRandom: letters and digits mixed, and the characters spread as a random string's are. A
|
||||||
|
// sentence's words are separated, so only an unbroken run reaches here.
|
||||||
|
func looksRandom(s string) bool {
|
||||||
|
var letters, digits int
|
||||||
|
counts := map[rune]int{}
|
||||||
|
for _, r := range s {
|
||||||
|
counts[r]++
|
||||||
|
switch {
|
||||||
|
case unicode.IsLetter(r):
|
||||||
|
letters++
|
||||||
|
case unicode.IsDigit(r):
|
||||||
|
digits++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if letters == 0 || digits == 0 {
|
||||||
|
// One class only: a word, or a number. A long number of digits alone is a count or a time.
|
||||||
|
return letters > 0 && hasUpperAndLower(s) && entropy(counts, len(s)) >= 4.0
|
||||||
|
}
|
||||||
|
return entropy(counts, len(s)) >= 3.3
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasUpperAndLower(s string) bool {
|
||||||
|
return strings.IndexFunc(s, unicode.IsUpper) >= 0 && strings.IndexFunc(s, unicode.IsLower) >= 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func entropy(counts map[rune]int, n int) float64 {
|
||||||
|
e := 0.0
|
||||||
|
for _, c := range counts {
|
||||||
|
p := float64(c) / float64(n)
|
||||||
|
e -= p * math.Log2(p)
|
||||||
|
}
|
||||||
|
return e
|
||||||
|
}
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// content_rule.go is messenger's own content rule (modules/messenger/cmd/messenger/content.go, novox/hq ADR
|
||||||
|
// 0253 §3), copied here verbatim — byte for byte, only the file renamed, since Go allows nothing else to
|
||||||
|
// differ (novox/hq issue 868, review round 1). mesh-mail carries it at run time, not only in its tests: the
|
||||||
|
// ask it builds is checked against the real rule before it is published (send.go), so a subject or a first
|
||||||
|
// line the masking does not catch (an IP, a hardware address, a path-shaped reference, a hex hash, …) falls
|
||||||
|
// back to a safe explanation rather than refusing the whole ask. Messenger's content rule stays messenger's:
|
||||||
|
// this is a pinned copy, never a dependency on it, the way modules/networkmanager/cmd/uplink-tools/copies_test.go
|
||||||
|
// pins its copy of uplink.go.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestContentRuleCopyMatchesMessengers(t *testing.T) {
|
||||||
|
src := filepath.Join("..", "..", "..", "messenger", "cmd", "messenger", "content.go")
|
||||||
|
theirs, err := os.ReadFile(src)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("messenger's content.go is not beside this module (%v); nothing to pin against", err)
|
||||||
|
}
|
||||||
|
mine, err := os.ReadFile("content_rule.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(mine, theirs) {
|
||||||
|
t.Error("mesh-mail's copy of messenger's content.go (content_rule.go) differs from " + src +
|
||||||
|
": change this copy to match messenger's content rule, verbatim, whenever it changes")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// How mesh-mail is (novox/hq ADR 0359 §6). Its keeper's liveness is its unit's; its watching is judged here, by the
|
||||||
|
// watch process, once a minute, and left in a status file the bundle's tool `mail_health` reads:
|
||||||
|
//
|
||||||
|
// - waits, naming the own secret mail-credential-<account>, for an account whose password nobody gave yet (ADR 0283),
|
||||||
|
// so the condition is needs-operator and the gate passes;
|
||||||
|
// - unhealthy, naming the account and why, when its connection is down or its server has not answered for
|
||||||
|
// SilentAfter (five minutes) — an IDLE that went silent;
|
||||||
|
// - healthy otherwise.
|
||||||
|
//
|
||||||
|
// A fault beside a wait is unhealthy and says both.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthTool is the module's tool the node-engine asks how it is.
|
||||||
|
const HealthTool = "mail_health"
|
||||||
|
|
||||||
|
// WatchEvery is how often the watch judges; StatusStale how old its verdict may be before the tool says it stopped.
|
||||||
|
const (
|
||||||
|
WatchEvery = time.Minute
|
||||||
|
StatusStale = 3 * WatchEvery
|
||||||
|
// SignInGrace is how long a password just given may wait for its keeper to sign in before that is a fault.
|
||||||
|
SignInGrace = 3 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// Judge is the module's health from what the watch reads: the accounts setting (or why it cannot be read), whether the
|
||||||
|
// index serves, which passwords were given, and each account's state as its keeper said it.
|
||||||
|
func Judge(accounts []Account, accountsErr, readyErr error, given func(Account) bool, states map[string]AccountState, now time.Time) health.Answer {
|
||||||
|
if accountsErr != nil {
|
||||||
|
return health.Unhealthy(accountsErr.Error())
|
||||||
|
}
|
||||||
|
var problems, fine []string
|
||||||
|
var waits []health.Wait
|
||||||
|
if readyErr != nil {
|
||||||
|
problems = append(problems, "the index cannot serve: "+readyErr.Error())
|
||||||
|
}
|
||||||
|
for _, a := range accounts {
|
||||||
|
if !given(a) {
|
||||||
|
waits = append(waits, health.Wait{
|
||||||
|
Part: "the mail account " + a.Name,
|
||||||
|
Secret: a.Secret(),
|
||||||
|
What: "the password of the mail account " + a.Name,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if readyErr != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
st, said := states[a.Name]
|
||||||
|
switch {
|
||||||
|
case !said:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its keeper has said nothing of it", a.Name))
|
||||||
|
case st.State == AccountWaits:
|
||||||
|
if now.Sub(st.Said) > SignInGrace {
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its password was given, and its keeper has not signed in for %s",
|
||||||
|
a.Name, now.Sub(st.Said).Round(time.Second)))
|
||||||
|
} else {
|
||||||
|
fine = append(fine, a.Name+" is signing in")
|
||||||
|
}
|
||||||
|
case st.State == AccountFailing:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s is not watched: %s", a.Name, st.Why))
|
||||||
|
case st.Heard.IsZero():
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered since its keeper started", a.Name))
|
||||||
|
case now.Sub(st.Heard) > SilentAfter:
|
||||||
|
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered for %s (its IDLE on the inbox is silent)",
|
||||||
|
a.Name, now.Sub(st.Heard).Round(time.Second)))
|
||||||
|
default:
|
||||||
|
fine = append(fine, fmt.Sprintf("%s is %s (%s)", a.Name, st.State, st.Why))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
waitSaid := ""
|
||||||
|
if len(waits) > 0 {
|
||||||
|
var names []string
|
||||||
|
for _, w := range waits {
|
||||||
|
names = append(names, w.Secret)
|
||||||
|
}
|
||||||
|
waitSaid = "waits for its password: " + strings.Join(names, ", ")
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case len(problems) > 0:
|
||||||
|
why := strings.Join(problems, "; ")
|
||||||
|
if waitSaid != "" {
|
||||||
|
why += "; and it " + waitSaid
|
||||||
|
}
|
||||||
|
return health.Unhealthy(why, waits...)
|
||||||
|
case len(waits) > 0:
|
||||||
|
why := "nothing is wrong, and it " + waitSaid
|
||||||
|
if len(fine) > 0 {
|
||||||
|
why += "; " + strings.Join(fine, "; ")
|
||||||
|
}
|
||||||
|
return health.Waiting(why, waits...)
|
||||||
|
case len(accounts) == 0:
|
||||||
|
return health.Healthy("no mail account is declared, so nothing is watched")
|
||||||
|
}
|
||||||
|
return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; ")))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status is the watch's verdict as it leaves it: the answer, and when it looked.
|
||||||
|
type Status struct {
|
||||||
|
health.Answer
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeStatus(path string, s Status) error {
|
||||||
|
raw, err := json.Marshal(s)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tmp := path + ".next"
|
||||||
|
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.Rename(tmp, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadStatus is the watch's last verdict, as the tool answers it: unhealthy when there is none, when it cannot be read,
|
||||||
|
// or when it is older than StatusStale.
|
||||||
|
func ReadStatus(path string, now time.Time) health.Answer {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return health.Unhealthy("the watch has left no verdict yet: " + err.Error())
|
||||||
|
}
|
||||||
|
var s Status
|
||||||
|
if err := json.Unmarshal(raw, &s); err != nil {
|
||||||
|
return health.Unhealthy("the watch's verdict cannot be read")
|
||||||
|
}
|
||||||
|
if age := now.Sub(s.At); age > StatusStale {
|
||||||
|
return health.Unhealthy(fmt.Sprintf("the watch has not looked for %s (its last verdict: %s)", age.Round(time.Second), s.Why))
|
||||||
|
}
|
||||||
|
// Read through the SDK's reader, so a wait it cannot check is never excused.
|
||||||
|
a, err := health.Read(mustJSON(s.Answer))
|
||||||
|
if err != nil {
|
||||||
|
return health.Unhealthy("the watch's verdict cannot be read: " + err.Error())
|
||||||
|
}
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustJSON(v any) []byte {
|
||||||
|
raw, _ := json.Marshal(v)
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchOnce is one look: what the watch reads, judged.
|
||||||
|
func watchOnce(ctx context.Context, env Env, open func(context.Context) (Store, func(), error), now time.Time) health.Answer {
|
||||||
|
accounts, accountsErr := ReadAccounts(env.Accounts)
|
||||||
|
var readyErr error
|
||||||
|
states := map[string]AccountState{}
|
||||||
|
st, done, err := open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
readyErr = err
|
||||||
|
} else {
|
||||||
|
defer done()
|
||||||
|
if readyErr = st.Ready(ctx); readyErr == nil {
|
||||||
|
if states, err = st.States(ctx); err != nil {
|
||||||
|
readyErr = err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Judge(accounts, accountsErr, readyErr, func(a Account) bool { return SecretGiven(env.Secrets, a) }, states, now)
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
|
)
|
||||||
|
|
||||||
|
var judgedAt = time.Date(2026, 10, 14, 18, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
func given(names ...string) func(Account) bool {
|
||||||
|
return func(a Account) bool { return contains(names, a.Name) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Before the password is given, mail_health answers waits, naming the secret, which the node-engine reads as a wait
|
||||||
|
// the controller can excuse (ADR 0283 §1, §3): the family mail-credential-* is declared issued outside the mesh.
|
||||||
|
func TestItWaitsNamingTheSecretUntilThePasswordIsGiven(t *testing.T) {
|
||||||
|
a := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
|
||||||
|
if a.State != health.StateWaits || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-mesh" || a.Healthy {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
read, err := health.Read(mustJSON(a))
|
||||||
|
if err != nil || read.State != health.StateWaits {
|
||||||
|
t.Fatalf("the node-engine reads it as %+v (%v)", read, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(a.Waits[0].What, "@") || strings.Contains(a.Why, "@") {
|
||||||
|
t.Fatalf("the wait says an address: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unhealthy, naming the account and why, when its connection is down or its server silent for five minutes.
|
||||||
|
func TestItIsUnhealthyWhenAnAccountIsDownOrSilent(t *testing.T) {
|
||||||
|
acc := []Account{testAccount()}
|
||||||
|
down := map[string]AccountState{"mesh": {Account: "mesh", State: AccountFailing, Why: "its server mail.example cannot be reached", Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), down, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "the account mesh is not watched: its server") {
|
||||||
|
t.Fatalf("down: %+v", a)
|
||||||
|
}
|
||||||
|
silent := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Heard: judgedAt.Add(-SilentAfter - time.Second), Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), silent, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "IDLE on the inbox is silent") {
|
||||||
|
t.Fatalf("silent: %+v", a)
|
||||||
|
}
|
||||||
|
fine := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Why: "watching", Heard: judgedAt.Add(-time.Minute), Said: judgedAt}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), fine, judgedAt); a.State != health.StateHealthy || !a.Healthy {
|
||||||
|
t.Fatalf("fine: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(acc, nil, errors.New("the schema is behind"), given("mesh"), fine, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("an index that cannot serve: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(nil, errors.New("the accounts setting is refused"), nil, given(), nil, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("a setting it cannot read: %+v", a)
|
||||||
|
}
|
||||||
|
// A fault beside a wait is unhealthy and says both (ADR 0283 §1).
|
||||||
|
two := []Account{testAccount(), {Name: "gmail", Kind: KindIMAP, Address: "x@example.com", Incoming: Server{Host: "h", Port: 993, Security: "tls"}}}
|
||||||
|
a := Judge(two, nil, nil, given("mesh"), down, judgedAt)
|
||||||
|
if a.State != health.StateUnhealthy || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-gmail" || !strings.Contains(a.Why, "mail-credential-gmail") {
|
||||||
|
t.Fatalf("both: %+v", a)
|
||||||
|
}
|
||||||
|
// A password just given is signing in for a while; not for ever.
|
||||||
|
signing := map[string]AccountState{"mesh": {Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-time.Minute)}}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("signing in: %+v", a)
|
||||||
|
}
|
||||||
|
signing["mesh"] = AccountState{Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-SignInGrace - time.Minute)}
|
||||||
|
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("never signed in: %+v", a)
|
||||||
|
}
|
||||||
|
if a := Judge(nil, nil, nil, given(), nil, judgedAt); a.State != health.StateHealthy || !strings.Contains(a.Why, "nothing is watched") {
|
||||||
|
t.Fatalf("no account: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tool reads the watch's verdict, and says when the watch stopped looking.
|
||||||
|
func TestTheHealthToolReadsTheWatchsVerdictAndSaysWhenItIsStale(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "health.json")
|
||||||
|
if a := ReadStatus(path, judgedAt); a.State != health.StateUnhealthy {
|
||||||
|
t.Fatalf("no verdict: %+v", a)
|
||||||
|
}
|
||||||
|
waits := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
|
||||||
|
if err := writeStatus(path, Status{Answer: waits, At: judgedAt}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a := ReadStatus(path, judgedAt.Add(time.Minute)); a.State != health.StateWaits || a.Waits[0].Secret != "mail-credential-mesh" {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
if a := ReadStatus(path, judgedAt.Add(StatusStale+time.Second)); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "has not looked") {
|
||||||
|
t.Fatalf("stale: %+v", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// One look of the watch, end to end over the files the mesh writes: the accounts setting, the secrets directory and
|
||||||
|
// the index.
|
||||||
|
func TestTheWatchJudgesFromTheFilesAndTheIndex(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
|
||||||
|
writeFile(t, env.Accounts, stageOne)
|
||||||
|
st := newMemStore()
|
||||||
|
open := func(context.Context) (Store, func(), error) { return st, func() {}, nil }
|
||||||
|
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateWaits {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
giveSecret(t, dir, Account{Name: "mesh"}, "pw")
|
||||||
|
_ = st.SetState(context.Background(), AccountState{Account: "mesh", State: AccountCurrent, Heard: judgedAt, Said: judgedAt})
|
||||||
|
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateHealthy {
|
||||||
|
t.Fatalf("%+v", a)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(Status{Answer: health.Healthy("x"), At: judgedAt})
|
||||||
|
if !strings.Contains(string(raw), `"state":"healthy"`) || !strings.Contains(string(raw), `"at":`) {
|
||||||
|
t.Fatalf("the status file's shape: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,570 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The adapter for an account of the kind `imap` (novox/hq ADR 0359 §1, §2, §5). For each account it keeps one
|
||||||
|
// connection: IDLE on the inbox, and every PollEvery a look at every other folder. Each look compares what the server
|
||||||
|
// holds with the index and says each difference as an event: a message added, its flags changed, or removed. The first
|
||||||
|
// look at a folder indexes what is there without saying it — what the bus is told is what changed since the mesh
|
||||||
|
// started watching, never the history of a mailbox.
|
||||||
|
//
|
||||||
|
// **It only reads.** Every folder is opened read-only (EXAMINE), and every message is fetched with BODY.PEEK, so
|
||||||
|
// watching never marks a message seen. The client is used through reader alone, which has no command that flags,
|
||||||
|
// moves, copies, deletes, appends or sends (§5); a test holds the code to that.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/tls"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
"github.com/emersion/go-imap/v2/imapclient"
|
||||||
|
)
|
||||||
|
|
||||||
|
// How the keeper paces itself. IdleCycle is how long one IDLE lasts before it is ended and begun again, which proves
|
||||||
|
// the server still answers; SilentAfter is how long without an answer makes the account unhealthy (§6).
|
||||||
|
const (
|
||||||
|
IdleCycle = 2 * time.Minute
|
||||||
|
PollEvery = 5 * time.Minute
|
||||||
|
SilentAfter = 5 * time.Minute
|
||||||
|
RetryFirst = 30 * time.Second
|
||||||
|
RetryMost = 5 * time.Minute
|
||||||
|
FetchBatch = 25
|
||||||
|
// BatchBytes bounds the mail one fetch of bodies holds; SizeBatch is how many sizes one fetch asks.
|
||||||
|
BatchBytes = 64 << 20
|
||||||
|
SizeBatch = 500
|
||||||
|
)
|
||||||
|
|
||||||
|
// Batches cuts the uids, in order, into batches of at most most messages whose sizes add up to at most limit bytes. A
|
||||||
|
// message larger than limit is a batch of its own. A size not known counts as nothing: the server listed the message,
|
||||||
|
// and the fetch says what it holds.
|
||||||
|
func Batches(uids []uint32, sizes map[uint32]int64, limit int64, most int) [][]uint32 {
|
||||||
|
var out [][]uint32
|
||||||
|
var cur []uint32
|
||||||
|
var bytes int64
|
||||||
|
for _, u := range uids {
|
||||||
|
size := sizes[u]
|
||||||
|
if len(cur) > 0 && (len(cur) >= most || bytes+size > limit) {
|
||||||
|
out = append(out, cur)
|
||||||
|
cur, bytes = nil, 0
|
||||||
|
}
|
||||||
|
cur = append(cur, u)
|
||||||
|
bytes += size
|
||||||
|
}
|
||||||
|
if len(cur) > 0 {
|
||||||
|
out = append(out, cur)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func uidSet(uids []uint32) imap.UIDSet {
|
||||||
|
set := make([]imap.UID, len(uids))
|
||||||
|
for i, u := range uids {
|
||||||
|
set[i] = imap.UID(u)
|
||||||
|
}
|
||||||
|
return imap.UIDSetNum(set...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func orBytes(n, def int64) int64 {
|
||||||
|
if n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// reader is the part of an IMAP client the keeper uses: nothing in it changes a mailbox.
|
||||||
|
type reader interface {
|
||||||
|
Login(username, password string) *imapclient.Command
|
||||||
|
List(ref, pattern string, options *imap.ListOptions) *imapclient.ListCommand
|
||||||
|
Select(mailbox string, options *imap.SelectOptions) *imapclient.SelectCommand
|
||||||
|
Fetch(numSet imap.NumSet, options *imap.FetchOptions) *imapclient.FetchCommand
|
||||||
|
Idle() (*imapclient.IdleCommand, error)
|
||||||
|
Caps() imap.CapSet
|
||||||
|
Closed() <-chan struct{}
|
||||||
|
Logout() *imapclient.Command
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dialer opens a connection to an account's server, handing what the server says unasked to the handler.
|
||||||
|
type Dialer func(a Account, handler *imapclient.UnilateralDataHandler) (reader, error)
|
||||||
|
|
||||||
|
// DialServer is the dialer for the mesh: TLS from the first byte, or STARTTLS before signing in, as the account says.
|
||||||
|
func DialServer(a Account, handler *imapclient.UnilateralDataHandler) (reader, error) {
|
||||||
|
opts := &imapclient.Options{UnilateralDataHandler: handler, TLSConfig: &tls.Config{ServerName: a.Incoming.Host, MinVersion: tls.VersionTLS12}}
|
||||||
|
var c *imapclient.Client
|
||||||
|
var err error
|
||||||
|
switch a.Incoming.Security {
|
||||||
|
case "tls":
|
||||||
|
c, err = imapclient.DialTLS(a.Incoming.Address(), opts)
|
||||||
|
case "starttls":
|
||||||
|
c, err = imapclient.DialStartTLS(a.Incoming.Address(), opts)
|
||||||
|
default:
|
||||||
|
err = fmt.Errorf("the account %s's security %q signs in over nothing this version speaks", a.Name, a.Incoming.Security)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keeper watches the accounts and keeps the index.
|
||||||
|
type Keeper struct {
|
||||||
|
Store Store
|
||||||
|
Objects Objects
|
||||||
|
Dial Dialer
|
||||||
|
SecretsDir string
|
||||||
|
Logf func(format string, a ...any)
|
||||||
|
Now func() time.Time
|
||||||
|
|
||||||
|
// Paces, for the tests; the constants above when zero.
|
||||||
|
IdleCycle, PollEvery, RetryFirst time.Duration
|
||||||
|
// BatchBytes, for the tests; the constant when zero.
|
||||||
|
BatchBytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) now() time.Time {
|
||||||
|
if k.Now != nil {
|
||||||
|
return k.Now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) logf(format string, a ...any) {
|
||||||
|
if k.Logf != nil {
|
||||||
|
k.Logf(format, a...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orDefault(d, def time.Duration) time.Duration {
|
||||||
|
if d > 0 {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run watches every account until ctx ends.
|
||||||
|
func (k *Keeper) Run(ctx context.Context, accounts []Account) {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for _, a := range accounts {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(a Account) {
|
||||||
|
defer wg.Done()
|
||||||
|
k.Watch(ctx, a)
|
||||||
|
}(a)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (k *Keeper) say(ctx context.Context, a Account, state, why string, heard, synced time.Time) {
|
||||||
|
if err := k.Store.SetState(ctx, AccountState{Account: a.Name, State: state, Why: why, Heard: heard, Synced: synced, Said: k.now()}); err != nil {
|
||||||
|
k.logf("[mesh-mail] the account %s's state could not be kept: %v", a.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Watch keeps one account until ctx ends: waits while its secret is not given, signs in, watches, and on any fault
|
||||||
|
// says it, waits a little longer each time, and begins again.
|
||||||
|
func (k *Keeper) Watch(ctx context.Context, a Account) {
|
||||||
|
retry := orDefault(k.RetryFirst, RetryFirst)
|
||||||
|
var heard, synced time.Time
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
pw, given, err := ReadSecret(k.SecretsDir, a)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
k.say(ctx, a, AccountFailing, "its password was given and cannot be read: "+err.Error(), heard, synced)
|
||||||
|
case !given:
|
||||||
|
k.say(ctx, a, AccountWaits, "its password ("+a.Secret()+") has not been given", heard, synced)
|
||||||
|
default:
|
||||||
|
s := &session{k: k, a: a, wake: make(chan struct{}, 1), heard: heard, synced: synced}
|
||||||
|
err = s.run(ctx, pw)
|
||||||
|
heard, synced = s.heard, s.synced
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
k.say(ctx, a, AccountFailing, err.Error(), heard, synced)
|
||||||
|
k.logf("[mesh-mail] the account %s: %v; trying again in %s", a.Name, err, retry)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(retry):
|
||||||
|
}
|
||||||
|
if given && err != nil {
|
||||||
|
retry = min(retry*2, RetryMost)
|
||||||
|
} else {
|
||||||
|
retry = orDefault(k.RetryFirst, RetryFirst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// session is one connection to one account's server.
|
||||||
|
type session struct {
|
||||||
|
k *Keeper
|
||||||
|
a Account
|
||||||
|
c reader
|
||||||
|
wake chan struct{}
|
||||||
|
heard time.Time
|
||||||
|
synced time.Time
|
||||||
|
// state and why are what the session last said of the account, and told when it said it.
|
||||||
|
state, why string
|
||||||
|
told time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// answered is the server having answered: kept, and told at least once a minute, so a long first look at a large
|
||||||
|
// mailbox still shows the account alive to the health watch.
|
||||||
|
func (s *session) answered() {
|
||||||
|
s.heard = s.k.now()
|
||||||
|
if s.state != "" && s.heard.Sub(s.told) >= time.Minute {
|
||||||
|
s.tell(s.state, s.why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tell says the account's state.
|
||||||
|
func (s *session) tell(state, why string) {
|
||||||
|
s.state, s.why, s.told = state, why, s.k.now()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
s.k.say(ctx, s.a, state, why, s.heard, s.synced)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *session) poke() {
|
||||||
|
select {
|
||||||
|
case s.wake <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *session) run(ctx context.Context, password string) error {
|
||||||
|
handler := &imapclient.UnilateralDataHandler{
|
||||||
|
Expunge: func(uint32) { s.poke() },
|
||||||
|
Mailbox: func(*imapclient.UnilateralDataMailbox) { s.poke() },
|
||||||
|
Fetch: func(*imapclient.FetchMessageData) { s.poke() },
|
||||||
|
}
|
||||||
|
c, err := s.k.Dial(s.a, handler)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("its server %s cannot be reached: %v", s.a.Incoming.Host, err)
|
||||||
|
}
|
||||||
|
s.c = c
|
||||||
|
defer c.Close()
|
||||||
|
if err := c.Login(s.a.SignInAs(), password).Wait(); err != nil {
|
||||||
|
var said *imap.Error
|
||||||
|
if errors.As(err, &said) {
|
||||||
|
return fmt.Errorf("its server refused the sign-in (%s): its password may be wrong, and is given again at the desk", strings.TrimSpace(said.Text))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("signing in failed: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
s.tell(AccountSyncing, "signed in; looking at its folders")
|
||||||
|
folders, err := s.folders()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
idles := c.Caps().Has(imap.CapIdle) || c.Caps().Has(imap.CapIMAP4rev2)
|
||||||
|
var polled time.Time
|
||||||
|
for ctx.Err() == nil {
|
||||||
|
if polled.IsZero() || s.k.now().Sub(polled) >= orDefault(s.k.PollEvery, PollEvery) {
|
||||||
|
for _, f := range folders {
|
||||||
|
if err := s.k.syncFolder(ctx, c, s.a, f, s.answered); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
polled = s.k.now()
|
||||||
|
}
|
||||||
|
// The inbox last, so it is the folder selected while the session idles.
|
||||||
|
if err := s.k.syncFolder(ctx, c, s.a, "INBOX", s.answered); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
s.synced = s.k.now()
|
||||||
|
s.tell(AccountCurrent, fmt.Sprintf("watching the inbox and %d other folder(s)", len(folders)))
|
||||||
|
if err := s.wait(ctx, idles); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = c.Logout().Wait()
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait is one IDLE on the inbox, ended by what the server says unasked, by the cycle's end or by ctx. Ending it waits
|
||||||
|
// for the server's answer, which is what proves the connection alive.
|
||||||
|
func (s *session) wait(ctx context.Context, idles bool) error {
|
||||||
|
cycle := time.NewTimer(orDefault(s.k.IdleCycle, IdleCycle))
|
||||||
|
defer cycle.Stop()
|
||||||
|
if !idles {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-cycle.C:
|
||||||
|
case <-s.c.Closed():
|
||||||
|
return errors.New("its server closed the connection")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
idle, err := s.c.Idle()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("its server refused to IDLE on the inbox: %v", err)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
case <-s.wake:
|
||||||
|
case <-cycle.C:
|
||||||
|
case <-s.c.Closed():
|
||||||
|
}
|
||||||
|
if err := idle.Close(); err != nil {
|
||||||
|
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
|
||||||
|
}
|
||||||
|
if err := idle.Wait(); err != nil {
|
||||||
|
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// folders is what is watched besides the inbox: the account's list, or every folder the server can open.
|
||||||
|
func (s *session) folders() ([]string, error) {
|
||||||
|
if len(s.a.Folders) > 0 {
|
||||||
|
return append([]string(nil), s.a.Folders...), nil
|
||||||
|
}
|
||||||
|
listed, err := s.c.List("", "*", nil).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("its folders cannot be listed: %v", err)
|
||||||
|
}
|
||||||
|
s.answered()
|
||||||
|
var out []string
|
||||||
|
for _, l := range listed {
|
||||||
|
if strings.EqualFold(l.Mailbox, "INBOX") || hasAttr(l.Attrs, imap.MailboxAttrNoSelect) || hasAttr(l.Attrs, imap.MailboxAttrNonExistent) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, l.Mailbox)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasAttr(attrs []imap.MailboxAttr, want imap.MailboxAttr) bool {
|
||||||
|
for _, a := range attrs {
|
||||||
|
if strings.EqualFold(string(a), string(want)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// syncFolder brings the index of one folder to what the server holds, and says each difference, unless this is the
|
||||||
|
// first look at the folder.
|
||||||
|
func (k *Keeper) syncFolder(ctx context.Context, c reader, a Account, folder string, answered func()) error {
|
||||||
|
sel, err := c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s cannot be opened: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
known, seen, err := k.Store.Folder(ctx, a.Name, folder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if seen && known.UIDValidity != sel.UIDValidity {
|
||||||
|
k.logf("[mesh-mail] the account %s's folder %s was renumbered by its server; it is indexed afresh", a.Name, folder)
|
||||||
|
if err := k.Store.DropFolder(ctx, a.Name, folder); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
seen = false
|
||||||
|
}
|
||||||
|
server := map[uint32][]string{}
|
||||||
|
if sel.NumMessages > 0 {
|
||||||
|
all, err := c.Fetch(imap.UIDSet{{Start: 1, Stop: 0}}, &imap.FetchOptions{UID: true, Flags: true}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's messages cannot be listed: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, m := range all {
|
||||||
|
server[uint32(m.UID)] = flagStrings(m.Flags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
index, err := k.Store.Flags(ctx, a.Name, folder)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
say := func(e *Event) *Event {
|
||||||
|
if seen {
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Removed: in the index, no longer on the server.
|
||||||
|
for uid := range index {
|
||||||
|
if _, still := server[uid]; still {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if m.UID == 0 {
|
||||||
|
m = Message{Account: a.Name, Folder: folder, UID: uid, UIDValidity: sel.UIDValidity}
|
||||||
|
}
|
||||||
|
if err := k.Store.Remove(ctx, a.Name, folder, uid, say(removedEvent(m))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Changed: on both, with other flags.
|
||||||
|
var added []uint32
|
||||||
|
for uid, flags := range server {
|
||||||
|
before, indexed := index[uid]
|
||||||
|
if !indexed {
|
||||||
|
added = append(added, uid)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if sameStrings(before, flags) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m.Flags = flags
|
||||||
|
if err := k.Store.SetFlags(ctx, a.Name, folder, uid, flags, say(changedEvent(m, before, flags))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Added: on the server, not yet in the index. Their sizes first, then their bodies in batches bounded by bytes as
|
||||||
|
// well as by count, so one batch never holds more than BatchBytes of mail in memory; a message larger than that
|
||||||
|
// is fetched alone, and said. Fetched without marking them seen.
|
||||||
|
sort.Slice(added, func(i, j int) bool { return added[i] < added[j] })
|
||||||
|
sizes := map[uint32]int64{}
|
||||||
|
for start := 0; start < len(added); start += SizeBatch {
|
||||||
|
part := added[start:min(start+SizeBatch, len(added))]
|
||||||
|
got, err := c.Fetch(uidSet(part), &imap.FetchOptions{UID: true, RFC822Size: true}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's new messages cannot be measured: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, m := range got {
|
||||||
|
sizes[uint32(m.UID)] = m.RFC822Size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cap := orBytes(k.BatchBytes, BatchBytes)
|
||||||
|
for _, batch := range Batches(added, sizes, cap, FetchBatch) {
|
||||||
|
if len(batch) == 1 && sizes[batch[0]] > cap {
|
||||||
|
k.logf("[mesh-mail] the account %s's message %d in %s is %d bytes, more than a batch holds; it is fetched alone",
|
||||||
|
a.Name, batch[0], folder, sizes[batch[0]])
|
||||||
|
}
|
||||||
|
msgs, err := c.Fetch(uidSet(batch), &imap.FetchOptions{
|
||||||
|
UID: true, Flags: true, Envelope: true, RFC822Size: true, InternalDate: true,
|
||||||
|
BodySection: []*imap.FetchItemBodySection{{Peek: true}},
|
||||||
|
}).Collect()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the folder %s's new messages cannot be fetched: %v", folder, err)
|
||||||
|
}
|
||||||
|
answered()
|
||||||
|
for _, fm := range msgs {
|
||||||
|
m, err := k.index(ctx, a, folder, sel.UIDValidity, fm)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := k.Store.Add(ctx, m, say(addedEvent(m))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return k.Store.SetFolder(ctx, Folder{Account: a.Name, Name: folder, UIDValidity: sel.UIDValidity, Messages: len(server), Synced: k.now()})
|
||||||
|
}
|
||||||
|
|
||||||
|
// index is a fetched message as the index keeps it: its metadata, its text, and its attachments kept in the store.
|
||||||
|
func (k *Keeper) index(ctx context.Context, a Account, folder string, validity uint32, fm *imapclient.FetchMessageBuffer) (Message, error) {
|
||||||
|
m := Message{Account: a.Name, Folder: folder, UIDValidity: validity, UID: uint32(fm.UID), Size: fm.RFC822Size,
|
||||||
|
Flags: flagStrings(fm.Flags), Date: fm.InternalDate.UTC(), From: []Person{}, To: []Person{}, Attachments: []AttachmentRef{}}
|
||||||
|
if env := fm.Envelope; env != nil {
|
||||||
|
m.MessageID, m.Subject = env.MessageID, env.Subject
|
||||||
|
if len(env.InReplyTo) > 0 {
|
||||||
|
m.InReplyTo = env.InReplyTo[0]
|
||||||
|
}
|
||||||
|
if !env.Date.IsZero() {
|
||||||
|
m.Date = env.Date.UTC()
|
||||||
|
}
|
||||||
|
m.From, m.To, m.Cc = people(env.From), people(env.To), people(env.Cc)
|
||||||
|
}
|
||||||
|
var raw []byte
|
||||||
|
if len(fm.BodySection) > 0 {
|
||||||
|
raw = fm.BodySection[0].Bytes
|
||||||
|
}
|
||||||
|
parsed, err := ParseMessage(raw)
|
||||||
|
if err != nil {
|
||||||
|
k.logf("[mesh-mail] the account %s's message %d in %s is read in part: %v", a.Name, m.UID, folder, err)
|
||||||
|
}
|
||||||
|
m.Body = parsed.Text
|
||||||
|
for _, p := range parsed.Attachments {
|
||||||
|
ref, err := Keep(ctx, k.Objects, p.Data, p.Type, p.Name)
|
||||||
|
if err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
m.Attachments = append(m.Attachments, ref)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func people(as []imap.Address) []Person {
|
||||||
|
out := []Person{}
|
||||||
|
for _, a := range as {
|
||||||
|
if addr := a.Addr(); addr != "" {
|
||||||
|
out = append(out, Person{Name: a.Name, Address: addr})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func flagStrings(fs []imap.Flag) []string {
|
||||||
|
out := make([]string, 0, len(fs))
|
||||||
|
for _, f := range fs {
|
||||||
|
if strings.EqualFold(string(f), `\Recent`) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, string(f))
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameStrings(a, b []string) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if a[i] != b[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the events ---------------------------------------------------------------------------------------------------
|
||||||
|
// EventKey and Changes, the event's shape, now live in mailevent.go of their own (issue 800 part A).
|
||||||
|
|
||||||
|
// metadata is what an event says of a message: never its body, never an attachment's bytes (ADR 0359 §2).
|
||||||
|
func metadata(m Message) map[string]any {
|
||||||
|
atts := make([]map[string]any, 0, len(m.Attachments))
|
||||||
|
for _, r := range m.Attachments {
|
||||||
|
atts = append(atts, map[string]any{"bucket": r.Bucket, "key": r.Key, "size": r.Size, "type": r.Type, "name": r.Name})
|
||||||
|
}
|
||||||
|
return map[string]any{
|
||||||
|
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity,
|
||||||
|
"message-id": m.MessageID, "in-reply-to": m.InReplyTo, "date": m.Date.UTC().Format(time.RFC3339),
|
||||||
|
"from": m.From, "to": m.To, "cc": m.Cc, "subject": m.Subject, "size": m.Size, "flags": m.Flags,
|
||||||
|
"attachments": atts,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func addedEvent(m Message) *Event {
|
||||||
|
return &Event{Key: EventKey(m.Account, "added"), Body: metadata(m)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func changedEvent(m Message, before, after []string) *Event {
|
||||||
|
body := metadata(m)
|
||||||
|
body["changes"] = map[string]any{"flags": map[string]any{"before": before, "after": after}}
|
||||||
|
return &Event{Key: EventKey(m.Account, "changed"), Body: body}
|
||||||
|
}
|
||||||
|
|
||||||
|
func removedEvent(m Message) *Event {
|
||||||
|
return &Event{Key: EventKey(m.Account, "removed"), Body: map[string]any{
|
||||||
|
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity, "message-id": m.MessageID,
|
||||||
|
}}
|
||||||
|
}
|
||||||
@@ -0,0 +1,300 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// keeping starts a keeper on one account of the test server, its paces short, and stops it when the test ends.
|
||||||
|
func keeping(t *testing.T, s *mailServer, st Store, objects Objects, secrets string) {
|
||||||
|
t.Helper()
|
||||||
|
k := &Keeper{Store: st, Objects: objects, Dial: s.dialer(), SecretsDir: secrets, Logf: t.Logf,
|
||||||
|
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { k.Watch(ctx, testAccount()); close(done) }()
|
||||||
|
t.Cleanup(func() { cancel(); <-done })
|
||||||
|
}
|
||||||
|
|
||||||
|
func stateOf(st Store, account string) AccountState {
|
||||||
|
states, _ := st.States(context.Background())
|
||||||
|
return states[account]
|
||||||
|
}
|
||||||
|
|
||||||
|
// float64OrUint reads a uid from an event body whether it came from memory (uint32) or from postgres (float64).
|
||||||
|
type float64OrUint struct{ v any }
|
||||||
|
|
||||||
|
func (f float64OrUint) value() uint32 {
|
||||||
|
switch n := f.v.(type) {
|
||||||
|
case uint32:
|
||||||
|
return n
|
||||||
|
case float64:
|
||||||
|
return uint32(n)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func uidOf(e Event) uint32 { return float64OrUint{e.Body["uid"]}.value() }
|
||||||
|
|
||||||
|
func eventsNamed(st Store, key string) []Event {
|
||||||
|
evs, _ := st.Pending(context.Background(), 1000)
|
||||||
|
var out []Event
|
||||||
|
for _, e := range evs {
|
||||||
|
if e.Key == key {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func describe(st Store) string {
|
||||||
|
evs, _ := st.Pending(context.Background(), 1000)
|
||||||
|
var keys []string
|
||||||
|
for _, e := range evs {
|
||||||
|
keys = append(keys, e.Key)
|
||||||
|
}
|
||||||
|
s := stateOf(st, "mesh")
|
||||||
|
return "state " + s.State + " (" + s.Why + "); events " + strings.Join(keys, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The whole watch, over a real IMAP conversation (ADR 0359 §2, §3, §5): the first look indexes what is there and says
|
||||||
|
// nothing; then a new mail in the inbox is said within moments of its delivery, through IDLE, with its metadata and
|
||||||
|
// its attachment by reference, never its body; its flags changed are said; its removal is said; a mail in another
|
||||||
|
// folder is said at the next poll; and watching never marks a message seen.
|
||||||
|
func TestTheKeeperSaysEachChangeOnTheBusAndOnlyReads(t *testing.T) {
|
||||||
|
eachStore(t, func(t *testing.T, st Store) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
op := s.operator(t)
|
||||||
|
deliver(t, op, "INBOX", plainMail("before the watch", "an old letter"))
|
||||||
|
deliver(t, op, "Archive", plainMail("archived long ago", "an older letter"))
|
||||||
|
secrets := t.TempDir()
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
objects := newMemObjects()
|
||||||
|
keeping(t, s, st, objects, secrets)
|
||||||
|
|
||||||
|
eventually(t, 10*time.Second, "the first look", func() (bool, string) {
|
||||||
|
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
|
||||||
|
})
|
||||||
|
ctx := context.Background()
|
||||||
|
if hits, _ := st.Search(ctx, Query{Text: "letter", Accounts: []string{"mesh"}}); len(hits) != 2 {
|
||||||
|
t.Fatalf("the first look indexed %d messages, not 2", len(hits))
|
||||||
|
}
|
||||||
|
if evs, _ := st.Pending(ctx, 100); len(evs) != 0 {
|
||||||
|
t.Fatalf("the first look said %d event(s): the history of a mailbox is not news", len(evs))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A new mail with an attachment, delivered while the keeper idles on the inbox.
|
||||||
|
deliver(t, op, "INBOX", attachedMail("the invoice"))
|
||||||
|
var added Event
|
||||||
|
eventually(t, 5*time.Second, "the new mail said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.added")
|
||||||
|
if len(evs) == 1 {
|
||||||
|
added = evs[0]
|
||||||
|
}
|
||||||
|
return len(evs) == 1, describe(st)
|
||||||
|
})
|
||||||
|
body := added.Body
|
||||||
|
if body["subject"] != "the invoice" || body["folder"] != "INBOX" || body["account"] != "mesh" || body["message-id"] != "att@example.org" {
|
||||||
|
t.Fatalf("the event says %v", body)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(body)
|
||||||
|
for _, never := range []string{"Please find", "dGhlIGludm9pY2Ug", attachmentData} {
|
||||||
|
if strings.Contains(string(raw), never) {
|
||||||
|
t.Fatalf("the event carries the body or an attachment's bytes (%q): %s", never, raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sum := sha256.Sum256([]byte(attachmentData))
|
||||||
|
key := hex.EncodeToString(sum[:])
|
||||||
|
if !strings.Contains(string(raw), `"key":"`+key+`"`) || !strings.Contains(string(raw), `"name":"invoice.pdf"`) {
|
||||||
|
t.Fatalf("the event does not name the attachment by its SHA-256 %s: %s", key, raw)
|
||||||
|
}
|
||||||
|
if kept, _ := objects.Get(ctx, key); string(kept) != attachmentData {
|
||||||
|
t.Fatalf("the attachment store keeps %q under %s", kept, key)
|
||||||
|
}
|
||||||
|
uid := uidOf(added)
|
||||||
|
m, ok, _ := st.Get(ctx, "mesh", "INBOX", uid)
|
||||||
|
if !ok || !strings.Contains(m.Body, "Please find the invoice attached.") {
|
||||||
|
t.Fatalf("the index keeps the text for mail_read: %+v", m)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flagged by the operator elsewhere: said as changed, with the flags before and after.
|
||||||
|
if _, err := op.Select("INBOX", nil).Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagFlagged}, Silent: true}, nil).Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, 5*time.Second, "the flag said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.changed")
|
||||||
|
if len(evs) != 1 {
|
||||||
|
return false, describe(st)
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(evs[0].Body["changes"])
|
||||||
|
return strings.Contains(string(raw), `"after":["\\Flagged"]`), string(raw)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Never marked seen by the watching: the message is still unseen on the server.
|
||||||
|
fetched, err := op.Fetch(imap.UIDSetNum(imap.UID(uid)), &imap.FetchOptions{Flags: true, UID: true}).Collect()
|
||||||
|
if err != nil || len(fetched) != 1 {
|
||||||
|
t.Fatalf("%v %v", fetched, err)
|
||||||
|
}
|
||||||
|
for _, f := range fetched[0].Flags {
|
||||||
|
if f == imap.FlagSeen {
|
||||||
|
t.Fatal("watching marked the message seen")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deleted by the operator elsewhere: said as removed.
|
||||||
|
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagDeleted}, Silent: true}, nil).Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := op.Expunge().Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
eventually(t, 5*time.Second, "the removal said", func() (bool, string) {
|
||||||
|
evs := eventsNamed(st, "mail.mesh.message.removed")
|
||||||
|
return len(evs) == 1 && uidOf(evs[0]) == uid && evs[0].Body["message-id"] == "att@example.org", describe(st)
|
||||||
|
})
|
||||||
|
if _, ok, _ := st.Get(ctx, "mesh", "INBOX", uid); ok {
|
||||||
|
t.Fatal("the removed message is still indexed")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A mail in another folder is said at the next poll.
|
||||||
|
deliver(t, op, "Archive", plainMail("filed away", "a letter filed"))
|
||||||
|
eventually(t, 5*time.Second, "the archived mail said", func() (bool, string) {
|
||||||
|
for _, e := range eventsNamed(st, "mail.mesh.message.added") {
|
||||||
|
if e.Body["folder"] == "Archive" && e.Body["subject"] == "filed away" {
|
||||||
|
return true, ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, describe(st)
|
||||||
|
})
|
||||||
|
if s := stateOf(st, "mesh"); s.State != AccountCurrent || time.Since(s.Heard) > time.Minute {
|
||||||
|
t.Fatalf("the account is %+v", s)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// No password given: the account waits, and nothing is dialled. A wrong one: the account fails, saying the server
|
||||||
|
// refused the sign-in. Given later: it is picked up without a restart.
|
||||||
|
func TestTheKeeperWaitsForThePasswordAndSaysARefusal(t *testing.T) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
st := newMemStore()
|
||||||
|
secrets := t.TempDir()
|
||||||
|
keeping(t, s, st, newMemObjects(), secrets)
|
||||||
|
eventually(t, 5*time.Second, "the wait", func() (bool, string) {
|
||||||
|
got := stateOf(st, "mesh")
|
||||||
|
return got.State == AccountWaits && strings.Contains(got.Why, "mail-credential-mesh"), describe(st)
|
||||||
|
})
|
||||||
|
giveSecret(t, secrets, testAccount(), "not-the-password")
|
||||||
|
eventually(t, 5*time.Second, "the refusal", func() (bool, string) {
|
||||||
|
got := stateOf(st, "mesh")
|
||||||
|
return got.State == AccountFailing && strings.Contains(got.Why, "refused the sign-in"), describe(st)
|
||||||
|
})
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
eventually(t, 10*time.Second, "the sign-in", func() (bool, string) {
|
||||||
|
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// It only reads a mailbox (ADR 0359 §5): no code of the module, but its tests, issues a command that flags, moves,
|
||||||
|
// copies, deletes, appends, creates, renames or otherwise writes to one over IMAP, and every folder is opened
|
||||||
|
// read-only and every body peeked. mail_send is the one exception the operator's own Approve carves out (ADR 0361
|
||||||
|
// §2): it never touches a mailbox either — it speaks SMTP, in smtp.go alone, to the account's outgoing server, and
|
||||||
|
// only once the operator approved it (send.go).
|
||||||
|
func TestNothingButTheTestsWritesToAMailbox(t *testing.T) {
|
||||||
|
files, err := filepath.Glob("*.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`)
|
||||||
|
for _, f := range files {
|
||||||
|
if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" {
|
||||||
|
// Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
|
||||||
|
// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2).
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(f)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range writes.FindAllSubmatch(raw, -1) {
|
||||||
|
// os is a file of the module's own, never a mailbox; stdio is the bus SDK (main.go's hear subscribes to
|
||||||
|
// the router's own decided subject, exactly as mesh-knowledge does, novox/hq issue 790 review round 1)
|
||||||
|
// — its Subscribe has nothing to do with an IMAP mailbox's.
|
||||||
|
if string(m[1]) != "os" && string(m[1]) != "stdio" {
|
||||||
|
t.Errorf("%s calls %s, which writes to a mailbox", f, m[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(string(raw), "net/smtp") {
|
||||||
|
t.Errorf("%s speaks SMTP: only smtp.go does, and only on the operator's Approve (ADR 0361 §2)", f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
src, _ := os.ReadFile("imap.go")
|
||||||
|
if !strings.Contains(string(src), "&imap.SelectOptions{ReadOnly: true}") || !strings.Contains(string(src), "{{Peek: true}}") {
|
||||||
|
t.Fatal("folders are not opened read-only, or bodies not peeked")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fetch of bodies is bounded by bytes (issue 733): batches cut by size and by count, and a message larger than a
|
||||||
|
// batch holds alone.
|
||||||
|
func TestBatchesAreBoundedByBytesAndALargerMessageGoesAlone(t *testing.T) {
|
||||||
|
sizes := map[uint32]int64{1: 40, 2: 40, 3: 500, 4: 10, 5: 10, 6: 10}
|
||||||
|
got := Batches([]uint32{1, 2, 3, 4, 5, 6}, sizes, 100, 2)
|
||||||
|
want := [][]uint32{{1, 2}, {3}, {4, 5}, {6}}
|
||||||
|
if fmt.Sprint(got) != fmt.Sprint(want) {
|
||||||
|
t.Fatalf("%v, not %v", got, want)
|
||||||
|
}
|
||||||
|
if got := Batches(nil, nil, 100, 25); len(got) != 0 {
|
||||||
|
t.Fatalf("%v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Over a real conversation: with a cap of a few hundred bytes, a large mail is fetched alone and said so, and every
|
||||||
|
// mail is still indexed and said.
|
||||||
|
func TestTheKeeperFetchesUnderAByteCap(t *testing.T) {
|
||||||
|
s := newMailServer(t)
|
||||||
|
op := s.operator(t)
|
||||||
|
st := newMemStore()
|
||||||
|
secrets := t.TempDir()
|
||||||
|
giveSecret(t, secrets, testAccount(), testPassword)
|
||||||
|
var mu sync.Mutex
|
||||||
|
var said []string
|
||||||
|
k := &Keeper{Store: st, Objects: newMemObjects(), Dial: s.dialer(), SecretsDir: secrets,
|
||||||
|
Logf: func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() },
|
||||||
|
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond, BatchBytes: 600}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { k.Watch(ctx, testAccount()); close(done) }()
|
||||||
|
t.Cleanup(func() { cancel(); <-done })
|
||||||
|
eventually(t, 10*time.Second, "the first look", func() (bool, string) { return stateOf(st, "mesh").State == AccountCurrent, describe(st) })
|
||||||
|
deliver(t, op, "INBOX", plainMail("small one", "short"))
|
||||||
|
deliver(t, op, "INBOX", plainMail("large one", strings.Repeat("a long letter ", 200)))
|
||||||
|
deliver(t, op, "INBOX", plainMail("small two", "short"))
|
||||||
|
eventually(t, 5*time.Second, "three mails said", func() (bool, string) {
|
||||||
|
return len(eventsNamed(st, "mail.mesh.message.added")) == 3, describe(st)
|
||||||
|
})
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
alone := 0
|
||||||
|
for _, line := range said {
|
||||||
|
if strings.Contains(line, "fetched alone") {
|
||||||
|
alone++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if alone != 1 {
|
||||||
|
t.Fatalf("said %d message(s) fetched alone: %v", alone, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// What the tests stand on: an IMAP server in this process (go-imap's own in-memory server, speaking the protocol over
|
||||||
|
// a real socket), the index in memory always and on a real postgres when one can be had (MESH_TEST_POSTGRES, or a
|
||||||
|
// private server started here from the machine's initdb on a socket of its own, listening on no port), and the
|
||||||
|
// attachment store in memory.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/emersion/go-imap/v2"
|
||||||
|
"github.com/emersion/go-imap/v2/imapclient"
|
||||||
|
"github.com/emersion/go-imap/v2/imapserver"
|
||||||
|
"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
code := m.Run()
|
||||||
|
stopPostgres()
|
||||||
|
os.Exit(code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the IMAP server --------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
const (
|
||||||
|
testUser = "operator@example.org"
|
||||||
|
testPassword = "the-password"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mailServer struct {
|
||||||
|
addr string
|
||||||
|
srv *imapserver.Server
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMailServer is a server holding one user with an inbox and an Archive folder.
|
||||||
|
func newMailServer(t *testing.T) *mailServer {
|
||||||
|
t.Helper()
|
||||||
|
mem := imapmemserver.New()
|
||||||
|
user := imapmemserver.NewUser(testUser, testPassword)
|
||||||
|
for _, f := range []string{"INBOX", "Archive"} {
|
||||||
|
if err := user.Create(f, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mem.AddUser(user)
|
||||||
|
srv := imapserver.New(&imapserver.Options{
|
||||||
|
NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) {
|
||||||
|
return mem.NewSession(), nil, nil
|
||||||
|
},
|
||||||
|
InsecureAuth: true,
|
||||||
|
Caps: imap.CapSet{imap.CapIMAP4rev1: {}, imap.CapIMAP4rev2: {}},
|
||||||
|
})
|
||||||
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go func() { _ = srv.Serve(ln) }()
|
||||||
|
t.Cleanup(func() { _ = srv.Close() })
|
||||||
|
return &mailServer{addr: ln.Addr().String(), srv: srv}
|
||||||
|
}
|
||||||
|
|
||||||
|
// dialer reaches the test server without TLS, which only a test does: the mesh's dialer signs in over TLS alone.
|
||||||
|
func (s *mailServer) dialer() Dialer {
|
||||||
|
return func(_ Account, h *imapclient.UnilateralDataHandler) (reader, error) {
|
||||||
|
c, err := imapclient.DialInsecure(s.addr, &imapclient.Options{UnilateralDataHandler: h})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return c, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// operator is a second client, as the operator's mail program or another sender is: it appends, flags and deletes.
|
||||||
|
func (s *mailServer) operator(t *testing.T) *imapclient.Client {
|
||||||
|
t.Helper()
|
||||||
|
c, err := imapclient.DialInsecure(s.addr, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = c.Close() })
|
||||||
|
if err := c.Login(testUser, testPassword).Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func deliver(t *testing.T, c *imapclient.Client, folder, raw string) {
|
||||||
|
t.Helper()
|
||||||
|
raw = strings.ReplaceAll(raw, "\n", "\r\n")
|
||||||
|
cmd := c.Append(folder, int64(len(raw)), nil)
|
||||||
|
if _, err := cmd.Write([]byte(raw)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := cmd.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := cmd.Wait(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func plainMail(subject, body string) string {
|
||||||
|
return "From: Alice <alice@example.org>\nTo: Operator <operator@example.org>\nSubject: " + subject +
|
||||||
|
"\nMessage-Id: <" + strings.ReplaceAll(subject, " ", "-") + "@example.org>\nDate: Wed, 14 Oct 2026 10:00:00 +0000\n" +
|
||||||
|
"MIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8\n\n" + body + "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// attachmentData is an attachment's content, and attachedMail a message carrying it.
|
||||||
|
const attachmentData = "the invoice, in full"
|
||||||
|
|
||||||
|
func attachedMail(subject string) string {
|
||||||
|
return "From: Bob <bob@example.org>\nTo: operator@example.org\nSubject: " + subject + "\nMessage-Id: <att@example.org>\n" +
|
||||||
|
"Date: Wed, 14 Oct 2026 11:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=XYZ\n\n" +
|
||||||
|
"--XYZ\nContent-Type: text/plain; charset=utf-8\n\nPlease find the invoice attached.\n" +
|
||||||
|
"--XYZ\nContent-Type: application/pdf; name=invoice.pdf\nContent-Disposition: attachment; filename=invoice.pdf\n" +
|
||||||
|
"Content-Transfer-Encoding: base64\n\ndGhlIGludm9pY2UsIGluIGZ1bGw=\n--XYZ--\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- the account ------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
func testAccount() Account {
|
||||||
|
return Account{Name: "mesh", Kind: KindIMAP, Address: testUser, Incoming: Server{Host: "localhost", Port: 993, Security: "tls"}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func giveSecret(t *testing.T, dir string, a Account, pw string) {
|
||||||
|
t.Helper()
|
||||||
|
if err := os.WriteFile(SecretPath(dir, a), []byte(pw+"\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventually waits for a condition, failing with what it last saw.
|
||||||
|
func eventually(t *testing.T, within time.Duration, what string, ok func() (bool, string)) {
|
||||||
|
t.Helper()
|
||||||
|
deadline := time.Now().Add(within)
|
||||||
|
last := ""
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
var done bool
|
||||||
|
if done, last = ok(); done {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
|
}
|
||||||
|
t.Fatalf("%s did not happen within %s; last seen: %s", what, within, last)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- postgres ---------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
var (
|
||||||
|
pgOnce sync.Once
|
||||||
|
pgBase string
|
||||||
|
pgDir string
|
||||||
|
pgWhy string
|
||||||
|
pgCount int
|
||||||
|
pgMu sync.Mutex
|
||||||
|
)
|
||||||
|
|
||||||
|
func startPostgres() {
|
||||||
|
if u := os.Getenv("MESH_TEST_POSTGRES"); u != "" {
|
||||||
|
pgBase = u
|
||||||
|
return
|
||||||
|
}
|
||||||
|
initdb, err := exec.LookPath("initdb")
|
||||||
|
if err != nil {
|
||||||
|
pgWhy = "no postgres: MESH_TEST_POSTGRES is unset and the machine has no initdb"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgctl := filepath.Join(filepath.Dir(initdb), "pg_ctl")
|
||||||
|
dir, err := os.MkdirTemp("", "mmpg")
|
||||||
|
if err != nil {
|
||||||
|
pgWhy = err.Error()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgDir = dir
|
||||||
|
if out, err := exec.Command(initdb, "-D", dir+"/data", "-U", "postgres", "-A", "trust", "-E", "UTF8", "--no-locale").CombinedOutput(); err != nil {
|
||||||
|
pgWhy = fmt.Sprintf("initdb: %v: %s", err, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if out, err := exec.Command(pgctl, "-D", dir+"/data", "-l", dir+"/log", "-w", "-o",
|
||||||
|
"-k "+dir+" -c listen_addresses='' -F", "start").CombinedOutput(); err != nil {
|
||||||
|
pgWhy = fmt.Sprintf("pg_ctl start: %v: %s", err, out)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pgBase = "postgres://postgres@/postgres?host=" + dir
|
||||||
|
}
|
||||||
|
|
||||||
|
func stopPostgres() {
|
||||||
|
if pgDir == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if initdb, err := exec.LookPath("initdb"); err == nil {
|
||||||
|
_ = exec.Command(filepath.Join(filepath.Dir(initdb), "pg_ctl"), "-D", pgDir+"/data", "-m", "immediate", "stop").Run()
|
||||||
|
}
|
||||||
|
_ = os.RemoveAll(pgDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// freshDatabase is a database of its own for one test, or a skip saying why there is none.
|
||||||
|
func freshDatabase(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
pgOnce.Do(startPostgres)
|
||||||
|
if pgBase == "" {
|
||||||
|
t.Skipf("NOT TESTED ON POSTGRES: %s", pgWhy)
|
||||||
|
}
|
||||||
|
pgMu.Lock()
|
||||||
|
pgCount++
|
||||||
|
name := fmt.Sprintf("mail_%d_%d", os.Getpid(), pgCount)
|
||||||
|
pgMu.Unlock()
|
||||||
|
ctx := context.Background()
|
||||||
|
admin, err := OpenPgStore(ctx, pgBase)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer admin.Close()
|
||||||
|
if _, err := admin.pool.Exec(ctx, "create database "+name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u, err := url.Parse(pgBase)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
u.Path = "/" + name
|
||||||
|
return u.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func pgStore(t *testing.T) *PgStore {
|
||||||
|
t.Helper()
|
||||||
|
s, err := OpenPgStore(context.Background(), freshDatabase(t))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(s.Close)
|
||||||
|
if _, err := s.Prepare(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// eachStore runs a test over the index in memory and on postgres.
|
||||||
|
func eachStore(t *testing.T, run func(t *testing.T, s Store)) {
|
||||||
|
t.Run("memory", func(t *testing.T) { run(t, newMemStore()) })
|
||||||
|
t.Run("postgres", func(t *testing.T) { run(t, pgStore(t)) })
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user