Compare commits

..
Author SHA1 Message Date
mesh-tools 62ec68355c Merge pull request 'mesh-mail-notifier asks its own machine's node-notifier (issue 865)' (#324) from fix/865-notifier-asks-its-own-machine into main 2026-10-15 21:34:27 +00:00
mesh-tools 6a5e6510ca Merge pull request 'The clipboard service waits for an older daemon's lock, and the history sees the daemon (hq issues 860, 862)' (#323) from fix/860-862-the-clipboard-takes-over-quietly into main 2026-10-15 21:31:46 +00:00
jochen 3f97d2d50d mesh-mail-notifier: Send names its own machine's node-notifier (issue 865)
mesh/merge-gate head 3f97d2d50d57 merged into trunk 6a5e6510cab2 — pass: builds mesh-mail-notifier → g14, shanks; no bus step; every machine composes with …
mesh/repo-check head 3f97d2d50d57 merged into trunk 6a5e6510cab2 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-15 23:27:24 +02:00
jschoubben 6112233649 Probe clipmenud's lock with flock's default conflict status, which BusyBox shares (issue 862)
mesh/merge-gate head 6112233649d3 merged into trunk 8c2d7e5f97f1 — pass: builds clipmenu → g14, shanks; no bus step; every machine composes with the change…
mesh/repo-check head 6112233649d3 merged into trunk 8c2d7e5f97f1 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-15 23:27:19 +02:00
mesh-tools 8c2d7e5f97 Merge pull request 'mesh-mail-notifier: the watcher runs in its runtime, not a process that exits at once (issue 859)' (#322) from fix/859-notifier-runs-in-its-runtime into main 2026-10-15 21:21:12 +00:00
jschoubben 21ebbffed4 Count clipmenud when it runs as a bash script (issue 860)
mesh/merge-gate head 21ebbffed469 merged into trunk 8c2d7e5f97f1 — pass: builds clipmenu → g14, shanks; no bus step; every machine composes with the change…
mesh/repo-check head 21ebbffed469 merged into trunk 8c2d7e5f97f1 — fail: its merge-check.sh failed: --- FAIL: TestTheLauncherWaitsQuietlyWhileAnotherClipme…
mesh/delivery superseded: a newer head of the same pull request
2026-10-15 23:18:22 +02:00
jochen 31120ed580 mesh-mail-notifier: watch retries a subscribe refusal instead of exiting (issue 859, review round 1)
mesh/merge-gate head 31120ed58029 merged into trunk c9848f4fb85c — pass: builds mesh-mail-notifier → g14, shanks; no bus step; every machine composes with …
mesh/repo-check head 31120ed58029 merged into trunk c9848f4fb85c — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A subscribe the runtime refuses no longer calls os.Exit(1), which took mail-notifier.status and
mail_notifier_health down with it: watch now retries with backoff (2s doubling to 1 minute), logging each
refusal, as mesh-knowledge's hear does. Also: the run() test proves liveness by having the watcher answer a
ping after run returns, instead of checking a channel only the test's own deferred cleanup could close; and
the manifest test is renamed to say what it checks (no process resource at all).
2026-10-15 23:17:14 +02:00
jschoubben cf304d8629 Wait quietly for clipmenud's session lock; count the operator's clipmenud (issues 860, 862) 2026-10-15 23:15:35 +02:00
jochen 12d7852082 mesh-mail-notifier: the watcher runs in the base invocation, not a separate notify process (issue 859)
The notify process's stdin is empty under systemd, so stdio.Serve returned at EOF, main returned, and
systemd restarted it every five seconds: the watch loop never ran long enough to show a notice. The watch
loop now starts beside stdio.Serve in the one process the node's runtime launches and keeps alive, as
mesh-knowledge's serve does. No resource declares a separate notify process any more.
2026-10-15 23:13:34 +02:00
mesh-tools c9848f4fb8 Merge pull request 'The clipboard keeps its history running and keeps it to itself (hq issues 743, 750)' (#320) from fix/743-750-the-clipboard-keeps-history-and-keeps-it-to-itself into main 2026-10-15 21:02:04 +00:00
jschoubben c070bea782 Declare how the clipboard daemon is ready, so the count of resources without health stays at 62 (ADR 0240 rule 8, issues 743, 750)
mesh/merge-gate head c070bea782aa merged into trunk 7443938fef69 — pass: builds clipmenu → g14, shanks; no bus step; every machine composes with the change…
mesh/repo-check head c070bea782aa merged into trunk 7443938fef69 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-15 22:59:27 +02:00
mesh-tools 7443938fef Merge pull request 'gitea_commit takes a large patch from a hash-checked spool file (issue 852)' (#319) from fix/852-commit-a-patch-file into main 2026-10-15 20:55:15 +00:00
mesh-tools aee97bf8ad Merge pull request 'The kernel module's transparent-huge-pages defaults to madvise (hq issue 856, ADR 0381)' (#321) from fix/856-the-kernel-module-runs-everywhere into main 2026-10-15 20:47:50 +00:00
jschoubben c816cafc00 The kernel module's transparent-huge-pages defaults to madvise, the mesh's chosen value (hq issue 856, ADR 0381 §2, §3)
mesh/merge-gate head c816cafc00bd merged into trunk de6657c846b9 — pass: builds kernel → ace, g14, novox, shanks; no bus step; every machine composes with …
mesh/repo-check head c816cafc00bd merged into trunk de6657c846b9 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-15 22:45:07 +02:00
jschoubben 2353350d79 The clipboard keeps its history running and keeps it to itself (hq issues 743, 750)
mesh/merge-gate out of date: judged against trunk aee97bf8ad42, the trunk is now 7443938fef69; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk aee97bf8ad42, the trunk is now 7443938fef69; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
clipmenud is declared as the operator's user unit, enabled and running, with a launcher that finds the display. Copy reads the clipboard back and fails with the reason. History and paste answer counts, times and lengths, never content.
2026-10-15 22:43:42 +02:00
mesh-toolsandClaude d4b8b24701 Let gitea_commit take a large patch from a hash-checked spool file (issue 852)
mesh/merge-gate head d4b8b2470159 merged into trunk aee97bf8ad42 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head d4b8b2470159 merged into trunk aee97bf8ad42 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A feature's diff is megabytes, too large to type into a call. gitea_commit now also takes patch_file, the
random hex name of a file in the module's spool directory /var/lib/mesh/forge-patches (mode 0733, owned by
the machine's account), with patch_sha256. The tool reads only that directory, never through a symlink or a
hard link, refuses a file whose SHA-256 differs, applies it as it applies patch, and removes it.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-10-15 20:39:13 +00:00
mesh-tools de6657c846 Merge pull request 'An active research effort takes new versions, reviewed as proposals (hq issue 844)' (#318) from fix/844-an-active-effort-takes-new-versions into main 2026-10-15 19:35:07 +00:00
jschoubben f75e1a91d9 Carry the proposal day only into the dated rules, so an old effort's marked insight stands (hq issue 844)
mesh/merge-gate head f75e1a91d957 merged into trunk a95b3fa053e0 — pass: builds mesh-knowledge → novox; no bus step; every machine composes with the change…
mesh/repo-check head f75e1a91d957 merged into trunk a95b3fa053e0 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-15 21:29:55 +02:00
jschoubben ebfec39937 Hold an active effort's new version to the dated rules of the day it is proposed (hq issue 844) 2026-10-15 21:26:55 +02:00
jschoubben 24d05701d3 An active research effort takes new versions, reviewed as proposals (hq issue 844) 2026-10-15 21:22:17 +02:00
mesh-tools a95b3fa053 Merge pull request 'Mail seats: mesh-mail declares them, mesh-mail-notifier shows new mail, messenger tells (issues 800, 797)' (#317) from feat/mail-seats into main 2026-10-15 18:04:19 +00:00
jochen c2cd2e1eb1 issue 800: fix review round 3 finding, a true sliding window instead of a session quota
mesh/merge-gate head c2cd2e1eb15e merged into trunk 789b0a9474c8 — pass: builds mesh-mail, messenger, new: modules/mesh-mail-notifier → novox; no bus step;…
mesh/repo-check head c2cd2e1eb15e merged into trunk 789b0a9474c8 — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check.sh): main's version judged it; the ch…
mesh/delivery delivered
mesh/delivery-group group feat/mail-seats delivered: every member is delivered
A session that only reset after a quiet minute kept counting, and so folding, a steady
trickle of mail forever even with no burst (one mail every 50s). N is now the account's
qualifying mail arrived in the trailing 60 seconds, counted fresh on every arrival; a fold
holds only its own overflow and the notice names just that count, not the whole history.
2026-10-15 19:03:05 +02:00
jochen aa62de2e78 issue 797: a dropped tell no longer crowds the conditions' cap out of its own history
A dropped tell was kept in the same bounded send history (KeptSends)
that the conditions' and notify's cap counts sends from. A flood of
drops could push real, still-recent sends out of that bounded history,
undercounting sentLastHour and letting the conditions' cap stop
holding. A dropped tell is still said once an hour in the log and
named in Tell's own "dropped" answer; it is simply never recorded into
that shared history any more.
2026-10-15 18:51:26 +02:00
jochen 66713c5a1e issue 800: fix review round 2 findings in mesh-mail-notifier
A buffer was replaced, discarding whatever it still held unsent, once
Window had passed since its first mail — losing a failed send waiting
for its retry, or a folded mail waiting for its minute's end, if a new
mail arrived meanwhile. The window now slides on the gap since the
last mail, not on time elapsed since the first: a session's count
never resets, and its buffer is never replaced, while anything in it
is still unsent.

The fold notice now counts every mail of the account in that minute,
not only the overflow, and says "mail" for one and "mails" for more.
2026-10-15 18:50:21 +02:00
jochen b771794f1c issue 800: keep notify's health as unit; the gate refuses a one-process module judged only by its own tool
ADR 0227 rule 8 / ADR 0240 rule 2 refuse a module with no health check of
another kind beside a tool check. mesh-mail-notifier has one process
(notify); wiring mail_notifier_health as its manifest health, as the review
note's wording suggested (mirroring mesh-mail's watch/mail_health), trips
that rule because mesh-mail also has keep (kind unit) beside watch.
mail_notifier_health stays built, tested and callable; notify's manifest
health stays kind unit until the lead decides how a one-process module
gets this automatically.
2026-10-15 18:33:22 +02:00
jochen 6c63a17cfa Merge branch 'feat/mail-seats' into ms-800-r1 2026-10-15 17:54:33 +02:00
jochen ed66a2e873 issue 800: fix review round 1 findings in mesh-mail-notifier
Flush now sorts accounts instead of walking a map, so notice order is
stable. A failed node-notifier.send keeps its mail pending and retries
at most once a minute, without holding up other accounts' notices, and
the failure is kept in the Store for a new health tool
(mail_notifier_health) that is unhealthy past ten minutes of failure.
The burst rule now follows the lead's reading of ADR 0378 S9: within an
account's sliding minute, the first three qualifying mails are shown
one by one and the rest are folded into one notice at the window's
end; there is no separate quiet-gap timer. The dedupe key is the
account and message id, falling back to folder/uidvalidity/uid only
when no message id is given, so a UIDVALIDITY reset cannot suppress a
new mail, and a redelivered event already buffered is never recounted.
status now lists every granted account, not only those with a notice,
and caches mail-store.accounts for a minute.
2026-10-15 17:54:27 +02:00
jochen cc73b38671 issue 797: tells no longer eat the conditions'/notify's cap, and a dropped tell is logged once an hour
Review round 1 of mesh-catalog found sentLastHour counting tells
against CapPerHour (ADR 0380 §8 says tells keep their own cap, never
shared), and the drop log firing on every drop instead of once an
hour per channel as the comment already said.
2026-10-15 17:46:39 +02:00
jochen 0993c31ad4 Merge branch 'feat/mail-seats' into ms-797 2026-10-15 17:37:22 +02:00
jochen 82475691b2 issue 797: messenger gains the tell verb for a plain notice
ADR 0380 §8: the operator channel's notify is shaped for conditions (a key that
stays open, escalation, a plain rule that rewrites a subject), so a module has
no way to say a plain notice such as new mail. tell sends a headline and one or
two lines at once to every channel, bypassing routing, the burst window and the
walk hold; it offers Details like any message, is kept in the history, but its
record is marked cleared at once so it never stays open. Its cap (10 a channel
an hour) is its own, apart from notify's and the conditions'.
2026-10-15 17:37:15 +02:00
jochen 89046e6613 mesh-mail's mail-notifier needs a list of node seats, as the controller reads it (issue 800) 2026-10-15 17:34:00 +02:00
jochen dc89995d96 Add mesh-mail-notifier, the default holder of mail-notifier (issue 800 part B)
ADR 0378 §9 as amended by ADR 0379: a module of its own, built from mesh-mail's
event shape, runs as its own account to hear mesh-mail's restricted MAIL stream
(ADR 0376 §2), shows each new inbox mail through node-notifier.send at most
once, folds more than three from one account into one notice, and emits its
own shown event.
2026-10-15 17:33:15 +02:00
jochen 6c10288f0b mesh-mail declares the mail seats and claims mail-store (issue 800 part A)
Declares mail-store and mail-webclient (held once for the mesh) and mail-notifier
(held once per machine, needing node-notifier) per ADR 0378 as amended by ADR 0379.
Claims mail-store and serves its four verbs through the same handlers as the
existing mail_* tools. Moves the event shape (EventKey, Changes) into its own
file, mailevent.go, so mesh-mail-notifier (issue 800 part B) can carry a
byte-identical copy. Adds checks/mailstore, refusing any manifest but mesh-mail's
own that claims mail-store (ADR 0378 §3), wired into merge-check.sh.
2026-10-15 17:16:53 +02:00
mesh-tools 789b0a9474 Merge pull request 'Mail and bus hardening: mesh-mail sends on Approve, desk prompt says what for, duplicate PR delivery, cpu_top (issues 790, 789, 783, 741, 811)' (#316) from feat/mail-hardening into main 2026-10-15 14:59:25 +00:00
jochen eb97a3d9a2 mesh-mail: refuse a send whose undecided index cannot be written, and never overwrite a corrupt index (issue 811, review round 3)
mesh/merge-gate head eb97a3d9a2d7 merged into trunk 0914a33fba5a — pass: builds disk-load, mesh-delivery, mesh-mail, rofi → ace, g14, novox, shanks; no bus…
mesh/repo-check head eb97a3d9a2d7 merged into trunk 0914a33fba5a — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/delivery-group group feat/mail-hardening delivered: every member is delivered
2026-10-15 15:40:16 +02:00
jochen 8b2e2300d8 mesh-mail: a send whose ask was never heard settles as ended, and the sweep reads only undecided sends (issue 811)
A send whose ask never reached the router (the publish failed, or it was otherwise never recorded) stayed kept and
undecided forever, so mail_send_status kept saying "waiting" and the minute sweep kept reading it. Settle it as
ended once its asked time plus SendAskLasts has passed, with the words that say why. Keep an index of undecided
send ids beside the sends (written when kept, removed once decided) so the sweep's work grows with undecided
sends only, not with every send ever kept.
2026-10-15 15:27:56 +02:00
jochen 2f2daea415 Merge branch 'feat/mail-hardening' into mh-790-r1 2026-10-15 15:08:21 +02:00
jochen 3b48f5962a mesh-mail: send on the operator's Approve itself, sweep, stale-send status, no header injection (issue 790, review round 1) 2026-10-15 15:07:46 +02:00
jochen 2b1678f315 disk-load: cpu_top's shares use the unrounded busy time, and key units by cgroup path (issue 741) 2026-10-15 15:06:06 +02:00
jochen ab739d02dd mesh-delivery: a merge that starts over a stopped duplicate is made from the forge's word too (issue 783) 2026-10-15 15:01:23 +02:00
jochen a60020b51a issue 741: merge feature branch head 2026-10-15 14:51:12 +02:00
jochen a8d84d1730 Add disk-load's cpu_top tool: CPU by systemd unit and container (issue 741)
No tool named which units or processes used CPU outside containers during
the novox overload (issue 739). cpu_top reads each cgroup's cpu.stat (no
root needed), ranks units or containers busiest first, and says the
machine's CPU in use and the busy time no unit accounts for.
2026-10-15 14:51:08 +02:00
jochen 9e8521afb5 mesh-delivery: a stopped duplicate's delivery follows the pull request that merges it (issue 783)
A delivery keyed by commit stayed stopped forever once an earlier, closed
duplicate pull request with the same head had stopped it, even though a
later pull request went on to merge that very commit: the merge only set
merged_as on the stopped record, and the walk that carried it waited for
this owner's word with no delivery left to give it. PullUpdated and
PullMerged now start such a delivery over, under the same id, for the
pull request that announces or merges it next, naming both pull requests
in the transition's why. Any other stop (a person's, a group's, the same
pull request's own close) is untouched.
2026-10-15 14:50:57 +02:00
jochen 8c60604eb7 Show what a desk's secret prompt is for, and declare mesh-mail's (issue 789)
The desk's prompt named only the module and the secret's key, so the operator
asking for one of mesh-mail's mail-credential-* accounts could not tell which
one. node-launcher.secret (modules/rofi) now shows the controller's optional
what as its own line under the module and secret names, escaped as text so
the catalogue's words never reach the prompt as markup; omitted as before
when none is declared. mesh-mail's mail-credential-* family now declares that
line with the long-form own-secrets shape the controller's manifest parser
reads (novox/hq issue 789 part b; part a landed in mesh-controller).
2026-10-15 14:28:45 +02:00
jochen 13cd710071 mesh-mail: add mail_send, asked and sent once on the operator's Approve (issue 790) 2026-10-15 13:57:47 +02:00
mesh-tools 0914a33fba Merge pull request 'mesh-mail: byte-bounded fetch, accounts only to granted callers, grants error kept, secret only stat'ed (hq issue 733)' (#315) from adr-359-mesh-mail-733 into main 2026-10-15 08:49:32 +00:00
jochen b5417c28d0 Merge remote-tracking branch 'origin/main' into adr-359-mesh-mail-733
mesh/merge-gate head b5417c28d026 merged into trunk 5f6a41146c3a — pass: builds mesh-mail, sent nowhere; no bus step; every machine composes with the chang…
mesh/repo-check head b5417c28d026 merged into trunk 5f6a41146c3a — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery covered after its stop: a later walk that contains it was delivered: plan-1792054175111921309 (novox/mesh-catalog at 0914a33f into main, do…
2026-10-15 10:44:37 +02:00
mesh-tools 5f6a41146c Merge pull request 'mesh-mail: the operator's mail accounts, watched and said on the bus — stage 1, own server read-only (hq ADR 0359)' (#300) from adr-359-mesh-mail into main 2026-10-15 08:44:07 +00:00
mesh-tools c71c513027 Merge pull request 'Read branch protection as the forge's administrator, so builds register again (issue 772, ADR 0375)' (#313) from 772-protection-read-as-admin into main 2026-10-15 00:22:49 +00:00
mesh-toolsandClaude 2e32f47e45 Read branch protection as the forge's administrator, so builds register again
mesh/merge-gate head 2e32f47e4596 merged into trunk 9492278d934b — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 2e32f47e4596 merged into trunk 9492278d934b — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery covered after its failure: a later walk that contains it was delivered: plan-1792097717281845096 (novox/mesh-catalog at 7443938f into main,…
The forge answers a repository's protection rules to its administrator
alone, and the tools' account no longer is one (ADR 0365), so every
delivery building a module failed at the controller's build-source check
with a 403. gitea_branch_protection_get now reads the rules with the
administrator account (ADR 0375); it only reads, and the protection set
and every write stay refused to the tools' account. Issue 772.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-10-15 00:18:39 +00:00
mesh-admin 9492278d93 Merge pull request 'Make the forge's tools act as an account of their own, not the administrator' (#311) from 365-forge-identities into main 2026-10-14 22:51:36 +00:00
mesh-admin 5ce2d07c7f Merge pull request 'Give a rollout the word of a stopped delivery that waits for it (hq issue 753)' (#312) from fix/753-a-rollout-is-never-left-without-a-word into main 2026-10-14 22:35:38 +00:00
jschoubben 373f339750 Give a walk the word of a stopped delivery that waits for it (hq issue 753)
mesh/merge-gate head 373f33975077 merged into trunk 7913b56d66f7 — pass: builds mesh-delivery → novox; no bus step; every machine composes with the change …
mesh/repo-check head 373f33975077 merged into trunk 7913b56d66f7 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A delivery its group stopped for a member delivered since waits for a later
walk of its repository that contains its merge (issue 747). When that walk
carried no living delivery, nothing gave it its word and it waited forever.
The delivery now takes its turn on the walk that answers its merge, in its
group's order. A walk owed such a word stays under the orphans' timer, so
past the grace without it `stalled` lists it with why (held, or the
controller's refusal), and a walk no living delivery holds names the ended
deliveries it carries.
2026-10-15 00:30:26 +02:00
mesh-adminandClaude Opus 5.5 48f0662fde Make the forge's tools act as an account of their own, not the administrator (issue 469, ADR 0365 clause 2)
mesh/merge-gate head 48f0662fdea2 merged into trunk 5ce2d07c7f47 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 48f0662fdea2 merged into trunk 5ce2d07c7f47 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
The tools' token reached everything the administrator may do, so their boundary rested on their own guards.
The gitea module now makes a plain account, mesh-tools, with the admin login, gives it write on code, issues
and pull requests of every novox repository through one team, and mints the tools' token for it; the admin's
old token goes only after the new one is kept. The Go forge bundle sends that token, never the admin login.
The forge shows protection rules only to a repository administrator, so every check reads a branch's
protection from the branch. mesh-admin stays a ready author, so marks made before the move keep counting.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 22:28:25 +00:00
mesh-admin 7913b56d66 Merge pull request 'Add the kernel module: transparent huge pages set per machine (hq issue 746, ADR 0369)' (#310) from fix/746-huge-pages-madvise into main 2026-10-14 21:52:49 +00:00
jschoubben ab1811d003 Reload, not restart, the huge pages unit when the setting changes (hq issue 746)
mesh/merge-gate head ab1811d00357 merged into trunk c79337579d6b — pass: builds new: modules/kernel, sent nowhere; no bus step; every machine composes with…
mesh/repo-check head ab1811d00357 merged into trunk c79337579d6b — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A restart ran restore then apply, so every change passed back through the
found mode and rotated the journal again. Reload applies the new value
directly; restore also drops its state; no rotation while the machine boots;
the unit is sandboxed.
2026-10-14 23:42:49 +02:00
mesh-admin c79337579d Merge pull request 'Make the forge's webhook a system one, and say whether its events are heard (hq issue 729)' (#309) from fix/729-the-forge-events-are-heard-or-said into main 2026-10-14 21:40:44 +00:00
jschoubben 3a84044ece Add the kernel module: transparent huge pages set per machine (hq issue 746)
With THP 'always' journald's mmapped journal files sit in 2 MiB folios, so each small
journal write dirties and writes back 2 MiB; on the control node that saturated the
root mirror holding the bus store and the store, and the controller lost its lease.
The module sets the mode from a per-machine setting (default always, the kernel's own,
so assigning it changes nothing until a machine sets madvise), verifies it took, rotates
the journal once so new files are mapped under the new mode, gives back what it found
when stopped, and serves kernel_huge_pages to measure file_thp per unit.
2026-10-14 23:31:13 +02:00
mesh-admin c51357f8af Merge pull request 'Cover a failed or stopped delivery once a later finished walk contains it (hq issue 747)' (#308) from fix/747-a-failed-member-is-covered-by-a-later-walk into main 2026-10-14 21:30:10 +00:00
jschoubben 2833a049aa Make the forge's webhook a system one, and say whether its events are heard (hq issue 729)
mesh/merge-gate head 2833a049aa0f merged into trunk c51357f8afd4 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 2833a049aa0f merged into trunk c51357f8afd4 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
POST /admin/hooks on gitea 1.27.3 makes a default webhook unless
config.is_system_webhook is "true": a default webhook fires for no existing
repository, and the system listing does not name it, so every start made
another and no event reached the receiver. Its edit never changes a secret
or a kind, so the new system webhook is made first and read back, then every
other hook at the receiver is deleted: system, default, and each
repository's copy of a default one, which would send its events twice.

gitea_events says where the receiver listens, the hook as ensured and what
it replaced, the verified and refused deliveries and the polls' pace. When
nothing verified for ten minutes, since the start or the last delivery,
while the poll found two heads or merges no event said, it is logged loudly
once per silent spell and answered unhealthy; so are deliveries refused
since the last verified one.
2026-10-14 23:28:41 +02:00
jschoubben 6f7f07635e Cover a failed or stopped delivery once a later finished walk contains it (hq issue 747)
mesh/merge-gate head 6f7f07635e02 merged into trunk 0f76b6e232a5 — pass: builds mesh-delivery → novox; no bus step; every machine composes with the change …
mesh/repo-check head 6f7f07635e02 merged into trunk 0f76b6e232a5 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
A group member whose walk failed in an outage stayed failed after a later walk
of its repository, containing its merge, was done; its group then stopped every
later walk of the other repository, and the catalogue could deliver nothing.

A failed or stopped delivery now reads covered when a done walk of its
repository, made after its merge, whose commit contains it (the forge's word,
as issue 362's containment), built every module it moves. A group counts it as
delivered, and a member the group stopped for it no longer holds a walk back:
the group stops a walk only for a member whose change is still undelivered.
A stop covered under issue 480 now likewise makes its group read reached, not
stopped.

While the forge is asked, the walk is held back rather than stopped, for at
most ten minutes from the first ask: a forge that keeps erroring or cannot say
then lets the walk be asked to leave the stopped merge out, as before. The
hold is said by waits, stalled and stalled walks. The pass reads kept walks
each tick, so today's stuck deliveries clear after a restart.
2026-10-14 23:19:06 +02:00
mesh-admin 0f76b6e232 Merge pull request 'Let an agent's session run the TypeScript check outside the toolchain image (issue 742)' (#307) from 742-ts-toolchain into main 2026-10-14 21:06:32 +00:00
mesh-adminandClaude Opus 5.5 3e16c055be Let an agent's session run the TypeScript check outside the toolchain image (issue 742)
mesh/merge-gate head 3e16c055be8e merged into trunk 0361c44a03a0 — pass: the change touches no module of the mesh's graph
mesh/repo-check head 3e16c055be8e merged into trunk 0361c44a03a0 — pass: THE CHANGE ALTERS ITS OWN CHECK (merge-check-typescript.sh): main's version judged…
mesh/delivery delivered
Outside the image, assemble the same node_modules in the check's own
workspace from mesh-tools' lock on the forge, so a module's TypeScript is
checked locally before a push.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:03:53 +00:00
mesh-admin 0361c44a03 Merge pull request 'Clear a forge token name held twice, and mint one process at a time (hq issue 740)' (#306) from fix/a-token-name-held-twice-is-cleared into main 2026-10-14 21:03:29 +00:00
jschoubben d1debdcce8 Clear a forge token name held twice, and mint one process at a time (hq issue 740)
mesh/merge-gate head d1debdcce816 merged into trunk 0e7d9e74d22a — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head d1debdcce816 merged into trunk 0e7d9e74d22a — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
The runtime starts the watcher and the tools as two processes over one kept
token. Both minted at once on 2026-10-14 and left two tokens named
mesh-tools; gitea refuses a delete by a name held twice, so every mint
failed and every forge tool with it. A replace now deletes each token by
the name by its id, from the account's whole list, and a mint holds a lock
file beside the kept token and reuses what another process minted.
2026-10-14 22:38:38 +02:00
jochen be53f5167a mesh-mail: bound a fetch by bytes, show mail_accounts only to its grantees, say why grants are unread, judge a secret without reading it (hq issue 733)
mesh/merge-gate out of date: judged against trunk 2130f9961291, the trunk is now 5f6a41146c3a; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 2130f9961291, the trunk is now 5f6a41146c3a; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
The review of #300 found a batch of 25 whole messages could hold a gigabyte,
mail_accounts told ungranted callers folder names, counts and server errors,
an unreadable grants setting was refused without its reason, and the watch
read the password only to learn it was there.
2026-10-14 22:24:11 +02:00
jochen 2130f99612 mesh-mail: say mail events on its own stream MAIL, granted per account by grants (hq ADR 0363)
mesh/merge-gate head 2130f9961291 merged into trunk c71c513027e5 — pass: builds new: modules/mesh-mail, sent nowhere; no bus step; every machine composes w…
mesh/repo-check head 2130f9961291 merged into trunk c71c513027e5 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
ADR 0359 asked for a stream of its own and a bus grant from the module's
grants; ADR 0363 built both. The events leave the shared EVENTS stream, and
one family pattern per change replaces the literal per-account emits, so a
new account changes no manifest.
2026-10-14 22:17:09 +02:00
jochen ed5690cee4 Merge remote-tracking branch 'origin/main' into adr-359-mesh-mail 2026-10-14 22:16:50 +02:00
jochen 70c1110c53 Add mesh-mail: watch the operator's own mailbox read-only and say each mail on the bus
mesh/merge-gate out of date: judged against trunk e22326240833, the trunk is now 482232e8c51c; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk e22326240833, the trunk is now 482232e8c51c; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
ADR 0359 decided one module holds the operator's mail accounts. This is its
first stage: an imap adapter that keeps IDLE on the inbox and polls the other
folders read-only, indexes each message in its own database, keeps each
attachment once by its SHA-256 in its bucket, and says every added, changed or
removed message as a metadata-only event. Bodies and attachments are served
only by its tools, to callers the trusted setting grants names. Its health
waits for the account's own secret until it is given at the desk (issue 710).
2026-10-14 20:14:42 +02:00
122 changed files with 14169 additions and 497 deletions
+3
View File
@@ -0,0 +1,3 @@
module git.novox.be/novox/mesh-catalog/checks/mailstore
go 1.22
+81
View File
@@ -0,0 +1,81 @@
// Command mailstore refuses any manifest but modules/mesh-mail claiming the mail-store seat (novox/hq ADR 0378
// §3, issue 800 part A): the operator's mail is read, searched and sent through this one seat, and a second
// claimant would be a second path to the same mail that ADR 0378 never accounted for.
//
// go run . <catalogue root>
//
// Exits 1 on a finding.
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
)
// MailStoreSeat is the seat only mesh-mail may claim.
const MailStoreSeat = "mail-store"
// MailStoreModule is the one module allowed to claim it.
const MailStoreModule = "mesh-mail"
// Finding is one manifest refused: a module other than mesh-mail claiming mail-store.
type Finding struct {
Module string
}
// Check reads every manifest under root/modules and answers which ones it refuses.
func Check(root string) ([]Finding, error) {
manifests, err := filepath.Glob(filepath.Join(root, "modules", "*", "module.json"))
if err != nil {
return nil, err
}
sort.Strings(manifests)
var out []Finding
for _, path := range manifests {
raw, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m struct {
Module string `json:"module"`
Claims []struct {
Name string `json:"name"`
} `json:"claims"`
}
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
if m.Module == MailStoreModule {
continue
}
for _, c := range m.Claims {
if c.Name == MailStoreSeat {
out = append(out, Finding{Module: m.Module})
break
}
}
}
return out, nil
}
func main() {
root := "."
if len(os.Args) > 1 {
root = os.Args[1]
}
found, err := Check(root)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
for _, f := range found {
fmt.Printf("%s claims the mail-store seat: only %s may (novox/hq ADR 0378 §3)\n", f.Module, MailStoreModule)
}
if len(found) > 0 {
os.Exit(1)
}
fmt.Println("mail-store: claimed only by mesh-mail")
}
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"os"
"path/filepath"
"testing"
)
func manifest(t *testing.T, root, module, claims string) {
t.Helper()
dir := filepath.Join(root, "modules", module)
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
body := `{"module":"` + module + `","claims":[` + claims + `]}`
if err := os.WriteFile(filepath.Join(dir, "module.json"), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
func TestAnotherModuleClaimingMailStoreIsRefused(t *testing.T) {
root := t.TempDir()
manifest(t, root, "mesh-mail", `{"name":"mail-store","scope":"mesh"}`)
manifest(t, root, "betterbird", `{"name":"mail-store","scope":"mesh"}`)
manifest(t, root, "dunst", `{"name":"node-notifier","scope":"node"}`)
found, err := Check(root)
if err != nil {
t.Fatal(err)
}
if len(found) != 1 || found[0].Module != "betterbird" {
t.Fatalf("refused %+v", found)
}
}
func TestTheCatalogueHasNone(t *testing.T) {
found, err := Check("../..")
if err != nil {
t.Fatal(err)
}
if len(found) != 0 {
t.Fatalf("%+v", found)
}
}
+42 -2
View File
@@ -23,9 +23,51 @@
# #
# By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads # By hand: MESH_CHECK_MODULES=gitea sh merge-check-typescript.sh, in the toolchain image (it reads
# /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same. # /app/node_modules), or with TOOLCHAIN_MODULES pointing at a node_modules that holds the same.
#
# **In an agent's session, outside the toolchain image** (novox/hq issue 742), the same node_modules is
# assembled first, in a throwaway directory under the check's own workspace, the way the image's `deps` stage
# assembles it: the toolchain's own package.json and lock (mesh-tools' node-tools/, read from the forge this
# clone came from, so the list is never copied here) installed with `npm ci`, then the SDK the mesh last
# published (`@novox/mesh-sdk@latest`), the @novox scope from the forge's package registry, read without a
# credential. Node and npm are the dev-tools module's; nothing is installed on the machine, no npm
# configuration or cache of the account's is read or written. Only when neither TOOLCHAIN_MODULES nor
# /app/node_modules is there: the forge's check, in the image, fetches nothing, as before. The forge is
# MESH_FORGE (default: the origin remote's), the owner MESH_FORGE_OWNER (default: the origin's owner).
set -eu set -eu
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
deps="${TOOLCHAIN_MODULES:-/app/node_modules}" deps="${TOOLCHAIN_MODULES:-/app/node_modules}"
if [ -z "${TOOLCHAIN_MODULES:-}" ] && [ ! -f "$deps/typescript/bin/tsc" ]; then
origin=$(git remote get-url origin 2>/dev/null || true)
forge="${MESH_FORGE:-$(printf '%s\n' "$origin" | sed -n 's#^\(https\{0,1\}://[^/]*\)/.*#\1#p')}"
owner="${MESH_FORGE_OWNER:-$(printf '%s\n' "$origin" | sed -n 's#^https\{0,1\}://[^/]*/\([^/]*\)/.*#\1#p')}"
if [ -z "$forge" ] || [ -z "$owner" ]; then
echo "NOT CHECKED: no TypeScript toolchain at $deps, and no forge to assemble one from (origin is '$origin'; set MESH_FORGE and MESH_FORGE_OWNER, or TOOLCHAIN_MODULES)"
exit 1
fi
echo "no TypeScript toolchain at $deps: assembling one from $forge/$owner/mesh-tools node-tools/ and $owner's package registry (novox/hq issue 742)"
tc="$work/toolchain"
mkdir -p "$tc"
printf '@%s:registry=%s/api/packages/%s/npm/\n' "$owner" "$forge" "$owner" >"$work/npmrc"
# Read with Node's own fetch (dev-tools declares Node, not curl); a status that is not 2xx fails.
if ! { node -e '
const [base, dir] = process.argv.slice(1);
Promise.all(["package.json", "package-lock.json"].map(async (f) => {
const r = await fetch(base + f);
if (!r.ok) throw new Error(base + f + ": " + r.status);
require("fs").writeFileSync(dir + "/" + f, await r.text());
})).catch((e) => { console.error(e.message); process.exit(1); });' \
"$forge/$owner/mesh-tools/raw/branch/main/node-tools/" "$tc" &&
(cd "$tc" && export npm_config_userconfig="$work/npmrc" npm_config_cache="$work/npm-cache" &&
npm ci --ignore-scripts --no-audit --no-fund --loglevel=error &&
npm install --ignore-scripts --no-audit --no-fund --no-save --loglevel=error "@$owner/mesh-sdk@latest"); }; then
echo "NOT CHECKED: the TypeScript toolchain could not be assembled from $forge (novox/hq issue 742)"
exit 1
fi
deps="$tc/node_modules"
fi
tsc="$deps/typescript/bin/tsc" tsc="$deps/typescript/bin/tsc"
if [ ! -f "$tsc" ]; then if [ ! -f "$tsc" ]; then
echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain" echo "NOT CHECKED: no TypeScript compiler at $tsc — this part runs in the TypeScript toolchain"
@@ -39,8 +81,6 @@ touched=$({
printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p' printf '%s\n' "${MESH_CHECK_CHANGED:-}" | tr ',' '\n' | sed -n 's#^modules/\([^/]*\)/.*#\1#p'
} | sed '/^$/d' | sort -u) } | sed '/^$/d' | sort -u)
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
failed="" failed=""
for m in $touched; do for m in $touched; do
d="modules/$m" d="modules/$m"
+3
View File
@@ -33,11 +33,14 @@
# 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could # 6. the secrets (novox/hq issue 342): no mergeable file fills a key with a vault secret that a setting could
# replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names # replace — the key is `protected`, or left out of the file's `takes-settings`. checks/secrets/known names
# the nested ones the controller cannot protect alone yet, and only shrinks. # the nested ones the controller cannot protect alone yet, and only shrinks.
# 7. the mail-store seat (novox/hq ADR 0378 §3, issue 800): no manifest but modules/mesh-mail claims it.
set -eu set -eu
(cd checks/words && go test -count=1 . && go run . ../..) (cd checks/words && go test -count=1 . && go run . ../..)
# The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it. # The agent's home (novox/hq ADR 0266): no archive, and no file written into, under it.
(cd checks/agenthome && go test -count=1 . && go run . ../..) (cd checks/agenthome && go test -count=1 . && go run . ../..)
# The mail-store seat (novox/hq ADR 0378 §3): only mesh-mail may claim it.
(cd checks/mailstore && go test -count=1 . && go run . ../..)
# The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The # The secrets (novox/hq issue 342): a password is not a setting, so no setting can replace the vault's. The
# known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused. # known list only shrinks: judged against the trunk's copy at the merge base, a line it does not hold is refused.
secrets_base="" secrets_base=""
+13 -7
View File
@@ -7,9 +7,15 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
- Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires - Claims the mesh's `node-clipboard` seat and serves its verbs `history` and `copy`. Requires
`x11-display` on its own machine. `x11-display` on its own machine.
- **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it. - **Declares `rofi-greenclip` absent** (ADR 0180). This module replaces it.
- Starts `clipmenud` **once per session**, from the session's start (the `xinitrc` slot `normal`). - Declares `clipmenud.service` as a **user unit of the operator's account**, enabled and running
It is not a user unit as well. Its packaged unit needs user-scoped units (mesh-host #72, not (`scope: user`, ADR 0177; issue 743), so `$mod+.` shows the history again. It is the only starter: the
merged), and the session start alone is one starter. session's `xinitrc` line is gone. A drop-in (`~/.config/systemd/user/clipmenud.service.d/mesh.conf`)
replaces `ExecStart` with the module's launcher, which finds the X display from its socket (the account's
service manager has no `DISPLAY`) and puts the text-only `xsel` first on the `PATH`; until the display is up
it waits (a quick exit would run into the start limit, which the drop-in also switches off, with `RestartSec=2s`). `DISPLAY` and `XAUTHORITY` come from the user manager when the session's xinitrc has handed them over (module xorg); with several X servers and no `DISPLAY` yet, it waits for them. The store is in the account's runtime directory (mode 0700),
so no other account reads it.
**After delivery the operator logs in again**: the running session keeps the `clipmenud` its xinitrc
started, and the new unit waits on clipmenud's session lock until that one ends.
- Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the - Binds `$mod+period` to `clipmenu`, as its own i3 drop-in (`50-clipmenu.conf`). clipmenu shows the
history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu. history through `dmenu`, the seat command of `node-launcher`, so it looks like every other menu.
- Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools - Its settings are environment contributions (ADR 0203), read by the daemon, the menu and the tools
@@ -23,11 +29,11 @@ The clipboard manager as a module (novox/hq ADR 0208, research 026/04).
| tool | does | | tool | does |
|---|---| |---|---|
| `node-clipboard.history` | the history, newest first, each entry once with its id, first line, time, size and text (cut) | | `node-clipboard.history` | the history, newest first: count, time and length of each entry, and whether the daemon collects. **Never a text, a first line or an id** (issue 750); the text is read through clipmenu on the operator's own screen |
| `node-clipboard.copy` | put text on the clipboard; it enters the history like any copy | | `node-clipboard.copy` | put text on the clipboard, then read it back; an error with the reason (no `DISPLAY`, `xsel` exited, other contents) when it does not hold the text. It enters the history like any copy |
| `clipmenu_paste` | what the clipboard holds now | | `clipmenu_paste` | how many bytes of text the clipboard holds now, never the text |
| `clipmenu_clear` | forget the history; the daemon's locks stay | | `clipmenu_clear` | forget the history; the daemon's locks stay |
| `clipmenu_delete` | forget one entry, by id or first line | | `clipmenu_delete` | forget one entry, by its place in the history (1 is the newest) and the time the history gave it, refused if the history changed since; not by text or id |
The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's The history is read from clipmenu's own store, in the account's runtime directory, under clipmenu's
own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an own lock, so a copy arriving meanwhile is neither lost nor half-written. `copy` hands the text to an
@@ -15,8 +15,7 @@ import (
) )
const ( const (
mostCopy = 1 << 20 mostCopy = 1 << 20
mostPaste = 64 << 10
// majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/. // majorVersion is clipmenu's store layout: <dir>/clipmenu.<major>.<user>/.
majorVersion = 6 majorVersion = 6
) )
@@ -71,15 +70,15 @@ func cksum(data []byte) string {
func entryID(line string) string { return cksum([]byte(line + "\n")) } func entryID(line string) string { return cksum([]byte(line + "\n")) }
// Entry is one clip in the history. // Entry is one clip in the history as a caller sees it: when and how long, never what. The text of a
// clip, its first line and its id (a checksum of that line) stay in clipmenu's store, readable on the
// operator's own screen through clipmenu (issue 750).
type Entry struct { type Entry struct {
ID string `json:"id"` At string `json:"at"`
Line string `json:"line"` Bytes int `json:"bytes"`
At string `json:"at"` id string
Bytes int `json:"bytes"` line string
Text string `json:"text,omitempty"` at int64
Truncated bool `json:"truncated,omitempty"`
at int64
} }
// HistoryResult is what node-clipboard.history answers. // HistoryResult is what node-clipboard.history answers.
@@ -119,14 +118,15 @@ func readStore(dir string) ([]Entry, error) {
} }
out := make([]Entry, 0, len(latest)) out := make([]Entry, 0, len(latest))
for line, ns := range latest { for line, ns := range latest {
out = append(out, Entry{ID: entryID(line), Line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)}) out = append(out, Entry{id: entryID(line), line: line, at: ns, At: time.Unix(0, ns).Format(time.RFC3339)})
} }
sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at }) sort.Slice(out, func(i, j int) bool { return out[i].at > out[j].at })
return out, scan.Err() return out, scan.Err()
} }
// History is the clipboard's history, newest first. // History is the clipboard's history, newest first: how many entries, when each was copied and how
func History(limit, maxBytes int) (HistoryResult, error) { // long each is, and whether the daemon is collecting. It never returns a copied text.
func History(limit int) (HistoryResult, error) {
dir, err := storeDir(findEnvironment()) dir, err := storeDir(findEnvironment())
if err != nil { if err != nil {
return HistoryResult{}, err return HistoryResult{}, err
@@ -140,15 +140,8 @@ func History(limit, maxBytes int) (HistoryResult, error) {
if i == limit { if i == limit {
break break
} }
if info, err := os.Stat(filepath.Join(dir, e.ID)); err == nil { if info, err := os.Stat(filepath.Join(dir, e.id)); err == nil {
e.Bytes = int(info.Size()) e.Bytes = int(info.Size())
if maxBytes > 0 {
raw, _ := os.ReadFile(filepath.Join(dir, e.ID))
if len(raw) > maxBytes {
raw, e.Truncated = raw[:maxBytes], true
}
e.Text = string(raw)
}
} }
out.Entries = append(out.Entries, e) out.Entries = append(out.Entries, e)
} }
@@ -159,6 +152,40 @@ func History(limit, maxBytes int) (HistoryResult, error) {
type CopyResult struct { type CopyResult struct {
Bytes int `json:"bytes"` Bytes int `json:"bytes"`
Unit string `json:"unit"` Unit string `json:"unit"`
// Verified says the clipboard was read back and held exactly the text.
Verified bool `json:"verified"`
}
// readBackWait is the pause between reads of the clipboard while the owner starts; read-back tries 15 times.
var readBackWait = 200 * time.Millisecond
// readBack waits for the clipboard to hold text and answers why when it does not.
func readBack(s Session, unit, text string) error {
var last string
for i := 0; i < 15; i++ {
time.Sleep(readBackWait)
r, err := s.run(3*time.Second, "", "xsel", "--output", "--clipboard")
switch {
case err != nil:
last = err.Error()
case r.Code != 0:
last = "xsel: " + strings.TrimSpace(r.Stderr)
case r.Stdout == text:
return nil
default:
last = fmt.Sprintf("it holds %d bytes of something else", len(r.Stdout))
}
}
why := ""
if s.Display == "" {
why = "; the session has no DISPLAY, so the owner could not reach the screen"
}
if j, err := s.run(5*time.Second, "", "journalctl", "--user", "-u", unit+".service", "-n", "5", "--no-pager", "-o", "cat"); err == nil {
if tail := strings.TrimSpace(j.Stdout); tail != "" {
why += "; the owner said: " + tail
}
}
return fmt.Errorf("the clipboard does not hold the text (%d bytes): %s%s", len(text), last, why)
} }
// Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the // Copy puts text on the clipboard. The clipboard is owned by a process until another copies, so the
@@ -194,17 +221,20 @@ func Copy(text string) (CopyResult, error) {
os.Remove(f.Name()) os.Remove(f.Name())
return CopyResult{}, err return CopyResult{}, err
} }
return CopyResult{Bytes: len(text), Unit: unit + ".service"}, nil if err := readBack(s, unit, text); err != nil {
return CopyResult{}, err
}
return CopyResult{Bytes: len(text), Unit: unit + ".service", Verified: true}, nil
} }
// PasteResult is what clipmenu_paste answers. // PasteResult is what clipmenu_paste answers: how much text the clipboard holds, never the text. The
// clipboard is the newest history entry, and a copied secret must not reach a caller (issue 750).
type PasteResult struct { type PasteResult struct {
Text string `json:"text"` Bytes int `json:"bytes"`
Bytes int `json:"bytes"` Text bool `json:"holds_text"`
Truncated bool `json:"truncated,omitempty"`
} }
// Paste reads the clipboard now. // Paste reads the clipboard now and says how long it is.
func Paste() (PasteResult, error) { func Paste() (PasteResult, error) {
s, err := findSession() s, err := findSession()
if err != nil { if err != nil {
@@ -217,11 +247,7 @@ func Paste() (PasteResult, error) {
if r.Code != 0 { if r.Code != 0 {
return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr)) return PasteResult{}, fmt.Errorf("xsel: %s", strings.TrimSpace(r.Stderr))
} }
out := PasteResult{Text: r.Stdout, Bytes: len(r.Stdout), Truncated: r.Truncated} return PasteResult{Bytes: len(r.Stdout), Text: len(r.Stdout) > 0}, nil
if len(out.Text) > mostPaste {
out.Text, out.Truncated = out.Text[:mostPaste], true
}
return out, nil
} }
// ChangeResult is what clear and delete answer. // ChangeResult is what clear and delete answer.
@@ -298,10 +324,12 @@ func Clear() (ChangeResult, error) {
return out, err return out, err
} }
// Delete forgets one entry, by id or by its first line. // Delete forgets one entry, by its place in the history as node-clipboard.history lists it (1 is the
func Delete(id, line string) (ChangeResult, error) { // newest) together with the time it reports, so a copy arriving since the listing cannot shift the
if (id == "") == (line == "") { // place onto another entry. Not by its text or id: either would let a caller confirm a guess of a copied line.
return ChangeResult{}, errors.New("give the entry's id or its line, one of the two") func Delete(place int, at string) (ChangeResult, error) {
if place < 1 {
return ChangeResult{}, errors.New("give the entry's place in the history, 1 for the newest")
} }
dir, err := storeOf() dir, err := storeOf()
if err != nil { if err != nil {
@@ -309,6 +337,17 @@ func Delete(id, line string) (ChangeResult, error) {
} }
var out ChangeResult var out ChangeResult
err = withStoreLock(dir, func() error { err = withStoreLock(dir, func() error {
entries, err := readStore(dir)
if err != nil {
return err
}
if place > len(entries) {
return fmt.Errorf("no entry %d: the clipboard history holds %d", place, len(entries))
}
if entries[place-1].At != at {
return fmt.Errorf("entry %d was copied at %s, not at %s: the history changed since it was listed; list it again", place, entries[place-1].At, at)
}
line := entries[place-1].line
raw, err := os.ReadFile(filepath.Join(dir, "line_cache")) raw, err := os.ReadFile(filepath.Join(dir, "line_cache"))
if err != nil && !errors.Is(err, os.ErrNotExist) { if err != nil && !errors.Is(err, os.ErrNotExist) {
return err return err
@@ -319,7 +358,7 @@ func Delete(id, line string) (ChangeResult, error) {
continue continue
} }
_, text, _ := strings.Cut(l, " ") _, text, _ := strings.Cut(l, " ")
if text == line || (id != "" && entryID(text) == id) { if text == line {
out.Removed++ out.Removed++
_ = os.Remove(filepath.Join(dir, entryID(text))) _ = os.Remove(filepath.Join(dir, entryID(text)))
continue continue
@@ -327,7 +366,7 @@ func Delete(id, line string) (ChangeResult, error) {
kept = append(kept, l) kept = append(kept, l)
} }
if out.Removed == 0 { if out.Removed == 0 {
return fmt.Errorf("no entry %s%s in the clipboard history", id, line) return fmt.Errorf("no entry %d in the clipboard history", place)
} }
content := strings.Join(kept, "\n") content := strings.Join(kept, "\n")
if content != "" { if content != "" {
@@ -1,6 +1,7 @@
package main package main
import ( import (
"encoding/json"
"errors" "errors"
"os" "os"
"os/exec" "os/exec"
@@ -51,46 +52,84 @@ func store(t *testing.T, clips map[string]string, order ...string) string {
return dir return dir
} }
func TestTheHistoryIsNewestFirstOnceEachWithItsText(t *testing.T) { func TestTheHistoryAnswersCountTimesAndLengthsNeverContent(t *testing.T) {
store(t, map[string]string{"first": "first", "second (2 lines)": "second\nline two"}, "first", "second (2 lines)", "first") store(t, map[string]string{"first": "first", "sk-ant-api03-SECRETKEY (2 lines)": "sk-ant-api03-SECRETKEY\nline two"}, "first", "sk-ant-api03-SECRETKEY (2 lines)", "first")
fakeProcess(t, nobody, "clipmenud") fakeProcess(t, nobody, "clipmenud")
h, err := History(10, 4096) h, err := History(10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if !h.Collecting || h.Total != 2 || h.Entries[0].Line != "first" || h.Entries[1].Text != "second\nline two" || h.Entries[1].Bytes != 15 { if !h.Collecting || h.Total != 2 || len(h.Entries) != 2 || h.Entries[0].Bytes != 5 || h.Entries[1].Bytes != 31 || h.Entries[0].At == "" {
t.Fatalf("%+v", h) t.Fatalf("%+v", h)
} }
if h, _ := History(1, 3); len(h.Entries) != 1 || h.Entries[0].Text != "fir" || !h.Entries[0].Truncated { raw, _ := json.Marshal(h)
t.Fatalf("limited and cut: %+v", h) for _, leak := range []string{"first", "SECRETKEY", "line two"} {
if strings.Contains(string(raw), leak) {
t.Fatalf("the history answers %q: %s", leak, raw)
}
} }
if h, _ := History(10, 0); h.Entries[0].Text != "" || h.Entries[0].Bytes != 5 { if h, _ := History(1); len(h.Entries) != 1 {
t.Fatalf("without text: %+v", h) t.Fatalf("limited: %+v", h)
} }
} }
func TestAnEntryIsDeletedByIdOrLineWithItsText(t *testing.T) { // at is the time History reports for the entry at place.
func at(t *testing.T, place int) string {
t.Helper()
h, err := History(100)
if err != nil || place > len(h.Entries) {
t.Fatalf("%+v, %v", h, err)
}
return h.Entries[place-1].At
}
func TestADeleteWhoseListingIsStaleIsRefused(t *testing.T) {
dir := store(t, map[string]string{"a": "a", "b": "b"}, "a", "b")
listed := at(t, 1) // b
f, _ := os.OpenFile(filepath.Join(dir, "line_cache"), os.O_APPEND|os.O_WRONLY, 0o600)
f.WriteString("1800000000000000000 a\n") // a copy arrives: place 1 is now a
f.Close()
if _, err := Delete(1, listed); err == nil || !strings.Contains(err.Error(), "list it again") {
t.Fatalf("the wrong entry was deleted: %v", err)
}
if h, _ := History(10); h.Total != 2 {
t.Fatalf("%+v", h)
}
}
func TestAnEntryIsDeletedByItsPlaceWithItsTextAndNeverByItsText(t *testing.T) {
dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a") dir := store(t, map[string]string{"a": "a", "b": "b", "c": "c"}, "a", "b", "c", "a")
r, err := Delete(entryID("a"), "") // newest first: a, c, b
r, err := Delete(1, at(t, 1))
if err != nil || r.Removed != 2 || r.Remaining != 2 { if err != nil || r.Removed != 2 || r.Remaining != 2 {
t.Fatalf("by id, both copies: %+v, %v", r, err) t.Fatalf("by place, both copies: %+v, %v", r, err)
} }
if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(filepath.Join(dir, entryID("a"))); !errors.Is(err, os.ErrNotExist) {
t.Fatal("the text stayed") t.Fatal("the text stayed")
} }
if r, err := Delete("", "b"); err != nil || r.Removed != 1 || r.Remaining != 1 { if r, err := Delete(2, at(t, 2)); err != nil || r.Removed != 1 || r.Remaining != 1 {
t.Fatalf("by line: %+v, %v", r, err) t.Fatalf("second place (b): %+v, %v", r, err)
} }
if _, err := Delete("", "zzz"); err == nil { if _, err := Delete(5, "x"); err == nil {
t.Fatal("a missing entry was reported deleted") t.Fatal("a missing place was reported deleted")
} }
if _, err := Delete("x", "y"); err == nil { if _, err := Delete(0, "x"); err == nil {
t.Fatal("both an id and a line were accepted") t.Fatal("place 0 was accepted")
} }
cache, _ := os.ReadFile(filepath.Join(dir, "line_cache")) cache, _ := os.ReadFile(filepath.Join(dir, "line_cache"))
if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 { if !strings.HasSuffix(string(cache), " c\n") || strings.Count(string(cache), "\n") != 1 {
t.Fatalf("line cache: %q", cache) t.Fatalf("line cache: %q", cache)
} }
for _, tool := range tools() {
if tool.Name == "clipmenu_delete" {
if _, ok := tool.Input["line"]; ok {
t.Fatal("delete still takes a line")
}
if _, ok := tool.Input["id"]; ok {
t.Fatal("delete still takes an id")
}
}
}
} }
func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) { func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
@@ -112,13 +151,21 @@ func TestClearForgetsEverythingButKeepsTheDaemonsLocks(t *testing.T) {
} }
} }
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) { func copyBed(t *testing.T, xsel string) string {
t.Helper()
store(t, nil) store(t, nil)
fakeProcess(t, nobody, "i3", "DISPLAY=:1") fakeProcess(t, nobody, "i3", "DISPLAY=:1")
bin := fakeBinaries(t, map[string]string{"systemctl": "true", "systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`}) bin := fakeBinaries(t, map[string]string{"systemctl": "true", "xsel": xsel,
"systemd-run": `echo "$*" > "$LOG"; for last; do :; done; cat "$last" > "$LOG.text"`})
t.Setenv("LOG", filepath.Join(bin, "log")) t.Setenv("LOG", filepath.Join(bin, "log"))
readBackWait = 0
return bin
}
func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManagerAndReadsItBack(t *testing.T) {
bin := copyBed(t, `cat "$LOG.text"`)
r, err := Copy("secret-free text") r, err := Copy("secret-free text")
if err != nil || r.Bytes != 16 || !strings.HasPrefix(r.Unit, "clipmenu-copy-") { if err != nil || r.Bytes != 16 || !r.Verified || !strings.HasPrefix(r.Unit, "clipmenu-copy-") {
t.Fatalf("%+v, %v", r, err) t.Fatalf("%+v, %v", r, err)
} }
asked, _ := os.ReadFile(filepath.Join(bin, "log")) asked, _ := os.ReadFile(filepath.Join(bin, "log"))
@@ -134,7 +181,23 @@ func TestCopyHandsTheTextToAnOwnerUnderTheAccountsServiceManager(t *testing.T) {
} }
} }
func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) { func TestCopyFailsWhenTheClipboardHoldsSomethingElse(t *testing.T) {
copyBed(t, `printf 'old contents'`)
_, err := Copy("new text")
if err == nil || !strings.Contains(err.Error(), "does not hold") || strings.Contains(err.Error(), "old contents") {
t.Fatalf("a copy that did not land was reported: %v", err)
}
}
func TestCopyFailsSayingWhyWhenTheOwnerCannotReachTheDisplay(t *testing.T) {
copyBed(t, `echo "xsel: Can't open display: (null)" >&2; exit 1`)
_, err := Copy("new text")
if err == nil || !strings.Contains(err.Error(), "Can't open display") {
t.Fatalf("the reason was not given: %v", err)
}
}
func TestPasteSaysHowMuchTheClipboardHoldsNeverWhat(t *testing.T) {
fakeMachine(t) fakeMachine(t)
if _, err := Paste(); !errors.Is(err, ErrNoSession) { if _, err := Paste(); !errors.Is(err, ErrNoSession) {
t.Fatal(err) t.Fatal(err)
@@ -142,8 +205,9 @@ func TestPasteReadsTheClipboardOrSaysThereIsNoSession(t *testing.T) {
fakeProcess(t, nobody, "i3", "DISPLAY=:1") fakeProcess(t, nobody, "i3", "DISPLAY=:1")
fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`}) fakeBinaries(t, map[string]string{"xsel": `[ "$*" = "--output --clipboard" ] && printf 'on the clipboard'`})
p, err := Paste() p, err := Paste()
if err != nil || p.Text != "on the clipboard" || p.Bytes != 16 { raw, _ := json.Marshal(p)
t.Fatalf("%+v, %v", p, err) if err != nil || p.Bytes != 16 || strings.Contains(string(raw), "on the clipboard") {
t.Fatalf("%s, %v", raw, err)
} }
} }
@@ -157,7 +221,26 @@ func TestWithoutAStoreTheChangesSayWhy(t *testing.T) {
if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") { if _, err := Clear(); err == nil || !strings.Contains(err.Error(), "clipmenud has not run") {
t.Fatal(err) t.Fatal(err)
} }
if h, err := History(5, 0); err != nil || h.Total != 0 || h.Collecting { if h, err := History(5); err != nil || h.Total != 0 || h.Collecting {
t.Fatalf("an empty history: %+v, %v", h, err) t.Fatalf("an empty history: %+v, %v", h, err)
} }
} }
// clipmenud is a bash script: its process is named "bash" and only its command line names clipmenud
// (issue 860: collecting was false while the daemon ran).
func TestCollectingSeesClipmenudRunAsABashScript(t *testing.T) {
store(t, map[string]string{"first": "first"}, "first")
fakeProcess(t, nobody, "bash")
cmdline := filepath.Join(procRoot, strconv.Itoa(nobody), "cmdline")
if err := os.WriteFile(cmdline, []byte("bash\x00/usr/bin/clipmenud\x00"), 0o644); err != nil {
t.Fatal(err)
}
if h, err := History(10); err != nil || !h.Collecting {
t.Fatalf("clipmenud runs as a bash script: %+v %v", h, err)
}
// another bash script is not the daemon
os.WriteFile(cmdline, []byte("bash\x00/usr/bin/other\x00"), 0o644)
if h, _ := History(10); h.Collecting {
t.Fatalf("a bash script that is not clipmenud counted: %+v", h)
}
}
+16 -18
View File
@@ -22,28 +22,23 @@ func tools() []stdio.Tool {
return []stdio.Tool{ return []stdio.Tool{
{ {
Name: "node-clipboard.history", Name: "node-clipboard.history",
Description: "What the operator copied, newest first: each entry's id, its first line, when, its size " + Description: "What the operator copied, newest first: how many entries, when each was copied and how long " +
"and its text (each cut at max_bytes). Whether the clipboard daemon is collecting.", "it is, never what it says. Whether the clipboard daemon is collecting. The text is read on the operator's own screen, through clipmenu.",
Input: map[string]any{ Input: map[string]any{
"limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"}, "limit": map[string]any{"type": "integer", "description": "at most this many entries (default 20, at most 500)"},
"max_bytes": map[string]any{"type": "integer", "description": "cut each entry's text at this many bytes; 0 leaves the text out (default 4096, at most 65536)"},
}, },
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
limit, err := whole(args, "limit", 20, 1, 500) limit, err := whole(args, "limit", 20, 1, 500)
if err != nil { if err != nil {
return nil, err return nil, err
} }
most, err := whole(args, "max_bytes", 4096, 0, 65536) return History(limit)
if err != nil {
return nil, err
}
return History(limit, most)
}, },
}, },
{ {
Name: "node-clipboard.copy", Name: "node-clipboard.copy",
Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " + Description: "Put text on the operator's clipboard, as if they had copied it; it enters the history " +
"like any copy. Answers how many bytes.", "like any copy. Reads the clipboard back and answers an error with the reason when it does not hold the text.",
Input: map[string]any{ Input: map[string]any{
"type": "object", "type": "object",
"properties": map[string]any{ "properties": map[string]any{
@@ -61,7 +56,7 @@ func tools() []stdio.Tool {
}, },
{ {
Name: "clipmenu_paste", Name: "clipmenu_paste",
Description: "What the operator's clipboard holds right now, as text (cut at 64 KiB, said in truncated).", Description: "How many bytes of text the operator's clipboard holds right now, never the text itself.",
Run: func(map[string]any) (any, error) { return Paste() }, Run: func(map[string]any) (any, error) { return Paste() },
}, },
{ {
@@ -71,19 +66,22 @@ func tools() []stdio.Tool {
}, },
{ {
Name: "clipmenu_delete", Name: "clipmenu_delete",
Description: "Forget one entry of the clipboard history, by its id as the history answers it, or by " + Description: "Forget one entry of the clipboard history, by its place in node-clipboard.history's list " +
"its first line exactly.", "(1 is the newest) and its time as the history gives it; refused when the two no longer match.",
Input: map[string]any{ Input: map[string]any{
"id": map[string]any{"type": "string", "description": "the entry's id"}, "place": map[string]any{"type": "integer", "description": "the entry's place in the history, 1 for the newest"},
"line": map[string]any{"type": "string", "description": "the entry's first line, exactly"}, "at": map[string]any{"type": "string", "description": "the entry's time, as the history reports it"},
}, },
Run: func(args map[string]any) (any, error) { Run: func(args map[string]any) (any, error) {
id, err := text(args, "id", false) place, err := whole(args, "place", 0, 0, 100000)
if err != nil { if err != nil {
return nil, err return nil, err
} }
line, _ := args["line"].(string) at, err := text(args, "at", true)
return Delete(id, line) if err != nil {
return nil, err
}
return Delete(place, at)
}, },
}, },
} }
@@ -1,9 +1,16 @@
package main package main
import ( import (
"context"
"net"
"os"
"os/exec"
"reflect" "reflect"
"strings" "strings"
"sync"
"syscall"
"testing" "testing"
"time"
) )
// clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history // clipmenu's shape (novox/hq ADR 0208, research 026/04): it claims node-clipboard serving history
@@ -27,20 +34,48 @@ func TestItClaimsTheClipboardSeatServingHistoryAndCopy(t *testing.T) {
} }
} }
func TestTheDaemonStartsOnceFromTheSessionsStart(t *testing.T) { func TestTheDaemonIsTheOperatorsUserUnitEnabledAndRunning(t *testing.T) {
m := readManifest(t) m := readManifest(t)
if len(m.Shell) != 1 || m.Shell[0].For != "xinitrc" || m.Shell[0].Slot != "normal" { if len(m.Shell) != 0 {
t.Fatalf("%+v", m.Shell) t.Fatalf("a second start from the session: %+v", m.Shell)
} }
code := m.Shell[0].Code svc := m.resource(t, "daemon")
// Started once, through the module's text-only xsel (a selection holding an image is never read). want := map[string]any{"id": "daemon", "type": "service", "unit": "clipmenud.service", "scope": "user",
if strings.Count(code, "clipmenud &\n") != 1 || !strings.Contains(code, `PATH="/usr/local/lib/mesh-clipmenu:$PATH" clipmenud &`) || "user": "${machine:account}", "state": "running", "boot": "enabled", "reload-on": nil}
strings.Contains(code, "greenclip") { for k, v := range want {
t.Fatalf("%q", code) if k != "reload-on" && svc[k] != v {
t.Fatalf("daemon %s = %v, want %v: %v", k, svc[k], v, svc)
}
} }
for _, r := range m.Resources { // ADR 0240 rule 8: a long-running resource says how it is ready. The launcher keeps the unit active while
if r["type"] == "service" || r["type"] == "process" { // it waits for the display and execs clipmenud, so the unit's state is the daemon's own liveness.
t.Fatalf("a second start: %v", r) if h, _ := svc["health"].(map[string]any); h["kind"] != "unit" {
t.Fatalf("daemon health = %v, want kind unit", svc["health"])
}
dropin := m.resource(t, "daemon-dropin")
content, _ := dropin["content"].(string)
if dropin["path"] != "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf" ||
dropin["owner"] != "${machine:account}" ||
!strings.Contains(content, "ExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session") ||
strings.Contains(content, "greenclip") {
t.Fatalf("drop-in: %v", dropin)
}
launcher, _ := m.resource(t, "launcher")["content"].(string)
if !strings.Contains(launcher, "DISPLAY") || !strings.Contains(launcher, "/tmp/.X11-unix") ||
!strings.Contains(launcher, `PATH="/usr/local/lib/mesh-clipmenu:$PATH"`) || !strings.Contains(launcher, "exec clipmenud") {
t.Fatalf("launcher: %q", launcher)
}
// Waits for the display instead of exiting into the start limit; DISPLAY from the manager wins.
if !strings.Contains(launcher, "until [ -n \"$DISPLAY\" ]") || !strings.Contains(launcher, "sleep 1") ||
strings.Contains(launcher, "exit 1") || strings.Contains(launcher, "exit 0") {
t.Fatalf("the launcher must wait, not exit: %q", launcher)
}
if !strings.Contains(content, "StartLimitIntervalSec=0") || !strings.Contains(content, "RestartSec=2s") {
t.Fatalf("drop-in lacks the safety net: %q", content)
}
for k, v := range m.Environment.Variables {
if !strings.Contains(content, "Environment="+k+"="+v) {
t.Fatalf("the unit lacks %s=%s", k, v)
} }
} }
} }
@@ -64,3 +99,126 @@ func TestTheToolsAgreeWithTheManifest(t *testing.T) {
checkTheToolsAgree(t, m) checkTheToolsAgree(t, m)
checkNoSecretsOrInstallationNames(t) checkNoSecretsOrInstallationNames(t)
} }
// Run for real: DISPLAY from the manager is used at once; several sockets and no DISPLAY wait.
func TestTheLauncherUsesTheManagersDisplayAndWaitsOtherwise(t *testing.T) {
m := readManifest(t)
script, _ := m.resource(t, "launcher")["content"].(string)
dir := t.TempDir()
file := dir + "/launcher"
// clipmenud and the socket directory are faked: the script's own paths are rewritten.
body := strings.ReplaceAll(script, "/tmp/.X11-unix", dir+"/x11")
body = strings.Replace(body, "exec clipmenud", `echo "DISPLAY=$DISPLAY"`, 1)
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
t.Fatal(err)
}
os.MkdirAll(dir+"/x11", 0o755)
fakeBin := dir + "/bin"
os.MkdirAll(fakeBin, 0o755)
// the manager's environment: empty until the file appears
os.WriteFile(fakeBin+"/systemctl", []byte("#!/bin/sh\n[ -f \""+dir+"/env\" ] && cat \""+dir+"/env\"\nexit 0\n"), 0o755)
run := func(env ...string) (string, bool) {
ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond)
defer cancel()
cmd := exec.CommandContext(ctx, "sh", file)
cmd.Env = append([]string{"PATH=" + fakeBin + ":" + os.Getenv("PATH"), "HOME=" + dir}, env...)
out, _ := cmd.Output()
return strings.TrimSpace(string(out)), ctx.Err() == nil
}
if out, ok := run("DISPLAY=:7"); !ok || out != "DISPLAY=:7" {
t.Fatalf("with DISPLAY: %q %v", out, ok)
}
if _, ok := run(); ok {
t.Fatal("it exited with no X server")
}
for _, n := range []string{"X0", "X1"} {
l, _ := net.Listen("unix", dir+"/x11/"+n)
defer l.Close()
}
if _, ok := run(); ok {
t.Fatal("it chose one of two servers")
}
// the session hands the manager its display while the launcher waits
go func() {
time.Sleep(1200 * time.Millisecond)
os.WriteFile(dir+"/env", []byte("DISPLAY=:1\nOTHER=x\n"), 0o644)
}()
if out, ok := run(); !ok || out != "DISPLAY=:1" {
t.Fatalf("the display that arrived while waiting: %q %v", out, ok)
}
}
// Run for real: while another clipmenud of the account holds the session lock the launcher waits without
// a word, and starts clipmenud once the lock is free (issue 862).
func TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock(t *testing.T) {
if _, err := exec.LookPath("flock"); err != nil {
t.Skip("flock is missing here")
}
m := readManifest(t)
script, _ := m.resource(t, "launcher")["content"].(string)
dir := t.TempDir()
body := strings.Replace(script, "exec clipmenud", `echo "started"`, 1)
file := dir + "/launcher"
if err := os.WriteFile(file, []byte(body), 0o755); err != nil {
t.Fatal(err)
}
store := dir + "/cm/clipmenu.6.op"
os.MkdirAll(store, 0o700)
lock := store + "/session_lock"
// the older clipmenud: it holds the lock the way clipmenud does, until it is killed
holder := exec.Command("flock", "-x", "-n", lock, "sleep", "60")
holder.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := holder.Start(); err != nil {
t.Fatal(err)
}
// flock's child shares the lock, so the whole group goes
release := func() { syscall.Kill(-holder.Process.Pid, syscall.SIGKILL); holder.Wait() }
defer release()
for i := 0; i < 50; i++ {
if probe := exec.Command("flock", "-n", lock, "true"); probe.Run() != nil {
break
}
time.Sleep(50 * time.Millisecond)
}
var out syncBuffer
cmd := exec.Command("sh", file)
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "DISPLAY=:7", "USER=op", "CM_DIR=" + dir + "/cm"}
cmd.Stdout, cmd.Stderr = &out, &out
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
done := make(chan error, 1)
go func() { done <- cmd.Wait() }()
select {
case <-done:
t.Fatalf("the launcher went on while the lock was held: %q", out.String())
case <-time.After(2500 * time.Millisecond):
}
if out.Len() != 0 {
t.Fatalf("the launcher spoke while it waited: %q", out.String())
}
release()
select {
case <-done:
case <-time.After(5 * time.Second):
cmd.Process.Kill()
t.Fatal("the launcher did not start clipmenud after the lock was freed")
}
if got := strings.TrimSpace(out.String()); got != "started" {
t.Fatalf("after the lock was freed: %q", got)
}
}
type syncBuffer struct {
mu sync.Mutex
b strings.Builder
}
func (s *syncBuffer) Write(p []byte) (int, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.b.Write(p)
}
func (s *syncBuffer) Len() int { s.mu.Lock(); defer s.mu.Unlock(); return s.b.Len() }
func (s *syncBuffer) String() string { s.mu.Lock(); defer s.mu.Unlock(); return s.b.String() }
+19 -1
View File
@@ -403,7 +403,7 @@ func processesOf(comm string) []int {
if owner, ok := ownerOf(dir); !ok || owner != uid { if owner, ok := ownerOf(dir); !ok || owner != uid {
continue continue
} }
if readTrimmed(filepath.Join(dir, "comm")) == comm { if readTrimmed(filepath.Join(dir, "comm")) == comm || runsScript(dir, comm) {
pids = append(pids, pid) pids = append(pids, pid)
} }
} }
@@ -411,6 +411,24 @@ func processesOf(comm string) []int {
return pids return pids
} }
// runsScript is whether the process is an interpreter running the script named name: clipmenud is a
// bash script, so its comm is "bash" and only its command line, "bash /usr/bin/clipmenud", names it.
func runsScript(procDir, name string) bool {
raw, err := os.ReadFile(filepath.Join(procDir, "cmdline"))
if err != nil {
return false
}
args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
if len(args) < 2 {
return false
}
switch filepath.Base(args[0]) {
case "bash", "sh", "dash", "zsh", "env":
return filepath.Base(args[1]) == name
}
return false
}
// signalAll sends sig to every process of this uid named comm, and answers the pids it reached. // signalAll sends sig to every process of this uid named comm, and answers the pids it reached.
func signalAll(comm string, sig syscall.Signal) []int { func signalAll(comm string, sig syscall.Signal) []int {
var reached []int var reached []int
+41 -7
View File
@@ -29,13 +29,6 @@
"CM_HISTLENGTH": "15" "CM_HISTLENGTH": "15"
} }
}, },
"shell": [
{
"for": "xinitrc",
"slot": "normal",
"code": "# The clipboard's history (module clipmenu, novox/hq ADR 0208): clipmenud collects every copy from\n# here on, once per session. It keeps the history in the account's runtime directory, so a reboot\n# forgets it, and with it every password that was ever copied.\n# Through the module's xsel, which reads only a selection offering text (see the README).\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\" clipmenud &\n"
}
],
"resources": [ "resources": [
{ {
"id": "package", "id": "package",
@@ -54,6 +47,47 @@
"path": "/usr/local/lib/mesh-clipmenu/xsel", "path": "/usr/local/lib/mesh-clipmenu/xsel",
"mode": "0755", "mode": "0755",
"content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n" "content": "#!/bin/sh\n# xsel as clipmenud sees it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# clipmenud records text, and reads a selection with `timeout 1 xsel -o`. A selection holding an\n# image (a screenshot copied as image/png) is sent in pieces; one second is too short for megabytes,\n# timeout kills xsel half way, and the program owning the image waits for ever for a reader that is\n# gone. From then on nothing can ask the clipboard anything: pastes hang, and an Electron app that\n# asks on its main thread freezes. So a read goes ahead only when the selection offers text.\ncase \" $* \" in\n*\" -o \"* | *\" --output \"*)\n\tselection=clipboard\n\tcase \" $* \" in\n\t*\" --primary \"* | *\" -p \"*) selection=primary ;;\n\t*\" --secondary \"* | *\" -s \"*) selection=secondary ;;\n\tesac\n\ttargets=$(timeout 1 xclip -selection \"$selection\" -t TARGETS -o 2>/dev/null) || exit 1\n\tprintf '%s\\n' \"$targets\" | grep -qxE 'UTF8_STRING|STRING|TEXT|text/plain(;charset=utf-8)?' || exit 1\n\t;;\nesac\nexec /usr/bin/xsel \"$@\"\n"
},
{
"id": "user-units-dir",
"type": "directory",
"path": "${machine:account-home}/.config/systemd/user",
"owner": "${machine:account}",
"mode": "0755"
},
{
"id": "daemon-dropins",
"type": "directory",
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d",
"owner": "${machine:account}",
"mode": "0755"
},
{
"id": "daemon-dropin",
"type": "file",
"path": "${machine:account-home}/.config/systemd/user/clipmenud.service.d/mesh.conf",
"owner": "${machine:account}",
"mode": "0644",
"content": "# clipmenud in the operator's account (module clipmenu, novox/hq ADR 0208). Owned by the mesh: replaced at every push.\n# The package's unit starts clipmenud with no DISPLAY; the module's launcher finds the display and puts the\n# text-only xsel first on the PATH. The settings repeat the module's environment, which the account's\n# service manager does not read.\n[Unit]\nStartLimitIntervalSec=0\n\n[Service]\nRestartSec=2s\nEnvironment=CM_SELECTIONS=clipboard\nEnvironment=CM_MAX_CLIPS=500\nEnvironment=CM_HISTLENGTH=15\nExecStart=\nExecStart=/usr/local/lib/mesh-clipmenu/clipmenud-session\n"
},
{
"id": "launcher",
"type": "file",
"path": "/usr/local/lib/mesh-clipmenu/clipmenud-session",
"mode": "0755",
"content": "#!/bin/sh\n# clipmenud as the operator's user unit starts it, written by the mesh (module clipmenu). Replaced at every push.\n#\n# The account's service manager starts before the X server does, and has a DISPLAY only once the session's\n# xinitrc has handed it over (module xorg). So this waits, quietly, until the display is known, and exits\n# only when the session ends with it: a quick exit would run the unit into its start limit. DISPLAY, when\n# the manager has it, wins. Without it, exactly one X socket names the display; none or several (a\n# second server beside the session) wait for the session's own to arrive.\n# A running process does not see what the session hands the manager later, so each pass asks the manager.\nuntil [ -n \"$DISPLAY\" ]; do\n\tfor pair in $(systemctl --user show-environment 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY)='); do\n\t\texport \"$pair\"\n\tdone\n\t[ -n \"$DISPLAY\" ] && break\n\tset -- /tmp/.X11-unix/X*\n\tif [ $# -eq 1 ] && [ -S \"$1\" ]; then\n\t\tDISPLAY=\":${1##*/X}\"\n\t\texport DISPLAY\n\telse\n\t\tsleep 1\n\tfi\ndone\nif [ -z \"$XAUTHORITY\" ] && [ -r \"$HOME/.Xauthority\" ]; then\n\tXAUTHORITY=\"$HOME/.Xauthority\"\n\texport XAUTHORITY\nfi\n# clipmenud allows one daemon per account: it takes an exclusive flock on its session_lock file and exits\n# with status 2 when that is held. After a delivery without a new login, the session's older clipmenud (from\n# its xinitrc) still holds it, and a start would run the unit round its restart loop. So this waits, quietly,\n# until the lock is free: it looks at the same file clipmenud locks (the directory is clipmenud's own rule)\n# and lets go at once. Exit status 1 is \"held\" for util-linux and BusyBox flock alike (no -E: BusyBox lacks it); any other outcome starts clipmenud, which says its own\n# error. Another daemon winning the gap between look and start costs one restart, not a loop.\nlock=\"${CM_DIR:-${XDG_RUNTIME_DIR-${TMPDIR-/tmp}}}/clipmenu.6.$USER/session_lock\"\nwhile [ -e \"$lock\" ]; do\n\tflock -n \"$lock\" true\n\t[ $? -eq 1 ] || break\n\tsleep 2\ndone\nPATH=\"/usr/local/lib/mesh-clipmenu:$PATH\"\nexport PATH\nexec clipmenud\n"
},
{
"id": "daemon",
"type": "service",
"unit": "clipmenud.service",
"scope": "user",
"user": "${machine:account}",
"state": "running",
"boot": "enabled",
"health": {
"kind": "unit"
}
} }
], ],
"build": { "build": {
+5 -2
View File
@@ -3,7 +3,9 @@
How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time How loaded a machine's storage is, read from `/proc` and `/sys` and nothing else: the share of CPU time
spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue spent waiting on I/O, I/O pressure stall, each disk's busy share, throughput, IOPS, wait and queue
depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue depth, the filesystems' space, and the units, containers and processes doing the I/O (novox/hq issue
312). It changes nothing and declares no resources, so it can be assigned to every machine. 312). It also reads a machine's CPU use by systemd unit and by container, from each cgroup's cpu.stat,
so an overload outside every container can be named (novox/hq issue 741). It changes nothing and
declares no resources, so it can be assigned to every machine.
## Why it exists, and why here ## Why it exists, and why here
@@ -22,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
## Tools ## Tools
All four only read. All five only read.
| tool | what | replaces | | tool | what | replaces |
|---|---|---| |---|---|---|
@@ -30,6 +32,7 @@ All four only read.
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` | | `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` | | `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` | | `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
### How each number is read ### How each number is read
+15 -6
View File
@@ -68,9 +68,10 @@ func (m *Machine) bottomDevices() map[string]bool {
return out return out
} }
// unitIO is every unit's storage I/O, by its cgroup's path: the innermost .service or .scope, because // unitPaths is every systemd unit's cgroup path under /sys/fs/cgroup: the innermost .service or
// io.stat counts a cgroup's descendants too and an outer unit would count them twice. // .scope, because a cgroup's own stat file counts its descendants too and an outer unit would count
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO { // them twice. Shared by every reader that walks the cgroup tree (unitIO here, unitCPU in cpu.go).
func (m *Machine) unitPaths() []string {
base := m.path("/sys/fs/cgroup") base := m.path("/sys/fs/cgroup")
var units []string var units []string
filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error { filepath.WalkDir(base, func(p string, d fs.DirEntry, err error) error {
@@ -89,11 +90,19 @@ func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
outer[dir] = true outer[dir] = true
} }
} }
out := map[string]cgroupIO{} out := make([]string, 0, len(units))
for _, u := range units { for _, u := range units {
if outer[u] { if !outer[u] {
continue out = append(out, u)
} }
}
return out
}
// unitIO is every unit's storage I/O, by its cgroup's path.
func (m *Machine) unitIO(counted map[string]bool) map[string]cgroupIO {
out := map[string]cgroupIO{}
for _, u := range m.unitPaths() {
text := m.read("/sys/fs/cgroup/" + u + "/io.stat") text := m.read("/sys/fs/cgroup/" + u + "/io.stat")
if text == "" { if text == "" {
continue continue
+176
View File
@@ -0,0 +1,176 @@
package main
import (
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
// cgroupCPU is a cgroup's processor time, from its cpu.stat.
type cgroupCPU struct {
usageUsec uint64
}
// parseCPUStat reads a cgroup's cpu.stat ("usage_usec 123\nuser_usec …\nsystem_usec …"), readable by
// every account, unlike /proc/<pid>/stat for another account's process.
func parseCPUStat(s string) cgroupCPU {
var c cgroupCPU
for _, line := range strings.Split(s, "\n") {
k, v, ok := strings.Cut(line, " ")
if !ok {
continue
}
if k == "usage_usec" {
c.usageUsec, _ = strconv.ParseUint(strings.TrimSpace(v), 10, 64)
}
}
return c
}
// unitCPU is every unit's processor time, by its cgroup's path. A cgroup whose cpu.stat cannot be
// read is left out, never an error for the whole answer.
func (m *Machine) unitCPU() map[string]cgroupCPU {
out := map[string]cgroupCPU{}
for _, u := range m.unitPaths() {
text := m.read("/sys/fs/cgroup/" + u + "/cpu.stat")
if text == "" {
continue
}
out[u] = parseCPUStat(text)
}
return out
}
// MachineCPU is the machine's CPU in use over the window, from /proc/stat's cpu line. CPUsBusy is
// rounded only for display; raw, unexported, is the same count unrounded, for topCPU's share and
// unaccounted math (review round 1 of novox/hq issue 741: a small busy time that rounds CPUsBusy to
// 0.0 must not zero out every unit's share along with it).
type MachineCPU struct {
CPUs int `json:"cpus"`
BusyPct float64 `json:"busy_pct"` // % of all CPUs together
CPUsBusy float64 `json:"cpus_busy"` // as a count of CPUs, e.g. 7.5 of 12, rounded for display
raw float64
}
// machineCPUOver is the machine's CPU in use between two readings of /proc/stat: everything but idle
// and iowait, counted against the window's total ticks over every CPU.
func machineCPUOver(a, b cpuTimes) MachineCPU {
total := float64(b.total() - a.total())
if total <= 0 {
return MachineCPU{CPUs: b.cpus}
}
idle := sub(b.idle, a.idle) + sub(b.iowait, a.iowait)
busy := total - idle
if busy < 0 {
busy = 0
}
raw := busy / total * float64(b.cpus)
return MachineCPU{CPUs: b.cpus, BusyPct: round1(busy / total * 100), CPUsBusy: round1(raw), raw: raw}
}
// CPUUsage is a unit's or a container's share of CPU between two readings of its cgroup's cpu.stat.
// Cgroup is its cgroup's path, relative to /sys/fs/cgroup, said beside Unit always (review round 1 of
// novox/hq issue 741): two cgroups can share a leaf name (an outer "init.scope" and a user session's
// "user.slice/user@1000.service/init.scope"), and the path is how they are told apart.
type CPUUsage struct {
Unit string `json:"unit,omitempty"`
Cgroup string `json:"cgroup,omitempty"`
Container string `json:"container,omitempty"`
CPUPct float64 `json:"cpu_pct"` // % of one CPU: 100 = one CPU busy the whole window
SharePct float64 `json:"share_pct"` // this unit's share of the machine's busy time
raw float64
}
// topCPU ranks the units, or the containers, with the most CPU time between two readings of
// cpu.stat, busiest first. unaccountedPct is the machine's busy time no unit accounts for, as a
// share of it (never below 0): the sum is always taken over every unit, whichever by ranks by, so a
// limit on the list never changes it.
func (m *Machine) topCPU(a, b map[string]cgroupCPU, window time.Duration, by string, limit int, cpusBusy float64) ([]CPUUsage, float64) {
windowUsec := window.Seconds() * 1e6
type entry struct {
path, unit, container string
pct float64
}
var entries []entry
var names map[string]string
var sumPct float64
for path, end := range b {
start, ok := a[path]
if !ok {
continue
}
d := sub(end.usageUsec, start.usageUsec)
if d == 0 {
continue
}
pct := d / windowUsec * 100
sumPct += pct
unit := filepath.Base(path)
var container string
if id := containerID.FindStringSubmatch("/" + path); id != nil {
if names == nil {
names = m.containerNames()
}
container = id[1][:12]
if n, ok := names[id[1]]; ok {
container = n
}
}
entries = append(entries, entry{path: path, unit: unit, container: container, pct: pct})
}
// Units are keyed by their cgroup's own path, never by its leaf name (filepath.Base): two cgroups can
// share a leaf name (an outer "init.scope" and a user session's nested one), and keying by the name
// alone would merge them into one row (review round 1 of novox/hq issue 741). Containers stay keyed
// by their name, as before.
groups := map[string]*CPUUsage{}
for _, e := range entries {
key := e.path
if by == "container" {
if e.container == "" {
continue
}
key = e.container
}
g, ok := groups[key]
if !ok {
g = &CPUUsage{Unit: e.unit, Cgroup: e.path, Container: e.container}
if by == "container" {
g.Unit, g.Cgroup = "", ""
}
groups[key] = g
}
g.raw += e.pct
}
out := make([]CPUUsage, 0, len(groups))
for _, g := range groups {
out = append(out, *g)
}
sort.Slice(out, func(i, j int) bool {
if out[i].raw != out[j].raw {
return out[i].raw > out[j].raw
}
return out[i].Unit+out[i].Cgroup+out[i].Container < out[j].Unit+out[j].Cgroup+out[j].Container
})
if len(out) > limit {
out = out[:limit]
}
for i := range out {
out[i].CPUPct = round1(out[i].raw)
if cpusBusy > 0 {
out[i].SharePct = round1(out[i].raw / (cpusBusy * 100) * 100)
}
}
unaccounted := cpusBusy*100 - sumPct
if unaccounted < 0 {
unaccounted = 0
}
unaccountedPct := 0.0
if cpusBusy > 0 {
unaccountedPct = round1(unaccounted / (cpusBusy * 100) * 100)
}
return out, unaccountedPct
}
+182
View File
@@ -0,0 +1,182 @@
package main
import (
"strings"
"testing"
"time"
)
// aCPUHog is a machine whose CPU is 75 % busy (1.5 of 2 CPUs) over a 2 s window: a unit, another unit,
// and a container, each with a known usage_usec delta, leaving part of the machine's busy time
// unaccounted — the shape of issue 739, where 750 % of 12 CPUs went to processes outside containers.
func aCPUHog(t *testing.T) *fake {
f := newFake(t)
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\ncpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\nintr 1\n")
f.then("/proc/stat", "cpu 100 0 50 50 0 0 0 0 0 0\ncpu0 50 0 25 25 0 0 0 0 0 0\ncpu1 50 0 25 25 0 0 0 0 0 0\nintr 2\n")
zero := "usage_usec 0\nuser_usec 0\nsystem_usec 0\n"
cg := func(path string, usec int) {
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", zero)
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\nuser_usec "+itoa(usec)+"\nsystem_usec 0\n")
}
cg("system.slice/alpha.service", 1000000) // 50 % of one CPU over 2 s
cg("system.slice/beta.service", 400000) // 20 %
cg("system.slice/docker-"+store+".scope", 600000) // 30 %, named "cpu-hog" by the runtime
f.answers["docker ps --no-trunc --format {{.ID}}\t{{.Names}}"] = store + "\tcpu-hog\n"
return f
}
func TestCPUTopRanksUnitsAndContainersByCPUOverTheWindow(t *testing.T) {
f := aCPUHog(t)
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
if m["by"] != "unit" || f.waited != 2*time.Second {
t.Errorf("the default: %+v, waited %s", m, f.waited)
}
cpu := m["cpu"].(MachineCPU)
if cpu.CPUs != 2 || cpu.BusyPct != 75 || cpu.CPUsBusy != 1.5 {
t.Errorf("machine cpu %+v", cpu)
}
top := m["top"].([]CPUUsage)
if len(top) != 3 {
t.Fatalf("three units moved: %+v", top)
}
if top[0].Unit != "alpha.service" || top[0].CPUPct != 50 || top[0].SharePct != 33.3 {
t.Errorf("busiest first: %+v", top[0])
}
if top[1].Container != "cpu-hog" || top[1].CPUPct != 30 || top[1].SharePct != 20 {
t.Errorf("the container, named by the runtime: %+v", top[1])
}
if top[2].Unit != "beta.service" || top[2].CPUPct != 20 || top[2].SharePct != 13.3 {
t.Errorf("the least busy: %+v", top[2])
}
if m["unaccounted_pct"] != 33.3 {
t.Errorf("unaccounted: %+v (machine busy 150%% of one CPU, units sum to 100%%, 50%% of 150%% left over)", m["unaccounted_pct"])
}
}
func TestCPUTopByContainerNamesOnlyContainers(t *testing.T) {
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "container", "seconds": 2.0})
if err != nil {
t.Fatal(err)
}
top := out.(map[string]any)["top"].([]CPUUsage)
if len(top) != 1 || top[0].Container != "cpu-hog" || top[0].Unit != "" || top[0].CPUPct != 30 {
t.Errorf("by container: %+v", top)
}
if _, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"by": "process"}); err == nil {
t.Error("by=process was accepted; cpu_top only ranks units and containers")
}
}
func TestCPUTopLimitsHowManyAreNamed(t *testing.T) {
out, err := tool(t, aCPUHog(t).machine(), "cpu_top").Run(map[string]any{"limit": 1.0, "seconds": 2.0})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
top := m["top"].([]CPUUsage)
if len(top) != 1 || top[0].Unit != "alpha.service" {
t.Errorf("limit 1: %+v", top)
}
// The limit narrows the list, never the unaccounted share, which is taken over every unit.
if m["unaccounted_pct"] != 33.3 {
t.Errorf("unaccounted with limit 1: %+v", m["unaccounted_pct"])
}
}
func TestCPUTopLeavesOutACgroupThatAppearsOrVanishesWithinTheWindow(t *testing.T) {
f := aCPUHog(t)
f.then("/sys/fs/cgroup/system.slice/short-lived.service/cpu.stat", "usage_usec 500000\n")
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
if err != nil {
t.Fatal(err)
}
top := out.(map[string]any)["top"].([]CPUUsage)
for _, u := range top {
if u.Unit == "short-lived.service" {
t.Errorf("a unit only in the second reading was counted: %+v", top)
}
}
if !strings.Contains(strings.Join(f.calls, ","), "docker ps") {
t.Errorf("the container runtime was never asked: %v", f.calls)
}
}
// aTinyLoad is a machine with 12 CPUs, only 0.04 of them busy over a 2 s window (busy 1 of 300 total
// ticks): rounded to one decimal for display, CPUsBusy reads 0.0, but a unit using half of that busy
// time still has a non-zero share — review round 1's decision that share_pct and unaccounted_pct are
// computed from the unrounded busy time, never from the rounded CPUsBusy.
func aTinyLoad(t *testing.T) *fake {
f := newFake(t)
cpus := "cpu0 0 0 0 0 0 0 0 0 0 0\ncpu1 0 0 0 0 0 0 0 0 0 0\ncpu2 0 0 0 0 0 0 0 0 0 0\ncpu3 0 0 0 0 0 0 0 0 0 0\n" +
"cpu4 0 0 0 0 0 0 0 0 0 0\ncpu5 0 0 0 0 0 0 0 0 0 0\ncpu6 0 0 0 0 0 0 0 0 0 0\ncpu7 0 0 0 0 0 0 0 0 0 0\n" +
"cpu8 0 0 0 0 0 0 0 0 0 0\ncpu9 0 0 0 0 0 0 0 0 0 0\ncpu10 0 0 0 0 0 0 0 0 0 0\ncpu11 0 0 0 0 0 0 0 0 0 0\n"
f.file("/proc/stat", "cpu 0 0 0 0 0 0 0 0 0 0\n"+cpus)
f.then("/proc/stat", "cpu 1 0 0 299 0 0 0 0 0 0\n"+cpus)
f.file("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 0\n")
f.then("/sys/fs/cgroup/system.slice/tiny.service/cpu.stat", "usage_usec 60000\n") // 3 % of one CPU over 2 s
return f
}
func TestCPUTopSharesAreNonZeroEvenWhenRoundedCPUsBusyIsZero(t *testing.T) {
out, err := tool(t, aTinyLoad(t).machine(), "cpu_top").Run(map[string]any{"seconds": 2.0})
if err != nil {
t.Fatal(err)
}
m := out.(map[string]any)
cpu := m["cpu"].(MachineCPU)
if cpu.CPUs != 12 || cpu.CPUsBusy != 0 {
t.Fatalf("the machine 0.04 CPUs busy, rounded for display: %+v", cpu)
}
top := m["top"].([]CPUUsage)
if len(top) != 1 || top[0].Unit != "tiny.service" || top[0].CPUPct != 3 || top[0].SharePct != 75 {
t.Fatalf("a non-zero share from the unrounded busy time: %+v", top)
}
if m["unaccounted_pct"] != 25.0 {
t.Fatalf("unaccounted from the unrounded busy time: %+v (want 25, so shares plus unaccounted add to 100)",
m["unaccounted_pct"])
}
}
// TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows is review round 1's other decision: units are keyed
// by their cgroup path, never by filepath.Base, so two cgroups that both end in "init.scope" — one at the
// root, one under a user's session — are two rows, not one merged by their shared leaf name.
func TestCPUTopKeepsCgroupsWithTheSameUnitNameAsSeparateRows(t *testing.T) {
f := aCPUHog(t)
cg := func(path string, usec int) {
f.file("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec 0\n")
f.then("/sys/fs/cgroup/"+path+"/cpu.stat", "usage_usec "+itoa(usec)+"\n")
}
cg("init.scope", 1000000) // the root init, 50 % of one CPU over 2 s
cg("user.slice/user@1000.service/init.scope", 400000) // a user session's init, 20 %
out, err := tool(t, f.machine(), "cpu_top").Run(map[string]any{"seconds": 2.0, "limit": 10.0})
if err != nil {
t.Fatal(err)
}
top := out.(map[string]any)["top"].([]CPUUsage)
var inits []CPUUsage
for _, u := range top {
if u.Unit == "init.scope" {
inits = append(inits, u)
}
}
if len(inits) != 2 {
t.Fatalf("two cgroups named init.scope, merged into %d row(s): %+v", len(inits), top)
}
byCgroup := map[string]CPUUsage{}
for _, u := range inits {
byCgroup[u.Cgroup] = u
}
if u, ok := byCgroup["init.scope"]; !ok || u.CPUPct != 50 {
t.Errorf("the root init.scope: %+v", byCgroup)
}
if u, ok := byCgroup["user.slice/user@1000.service/init.scope"]; !ok || u.CPUPct != 20 {
t.Errorf("the user session's init.scope: %+v", byCgroup)
}
}
+51
View File
@@ -85,6 +85,57 @@ func Tools(m *Machine) []stdio.Tool {
return out, nil return out, nil
}, },
}, },
{
Name: "cpu_top",
Description: "What used the most CPU over a few seconds, from each unit's cgroup cpu.stat, which needs no root: the machine's " +
"CPU in use (as % of all CPUs and as CPUs busy, from /proc/stat), then by=unit (the default) each systemd unit and " +
"container ranked by CPU, busiest first, with the busy time no unit accounts for; by=container only the containers, " +
"by name. Replaces top, systemd-cgtop. (r)",
Input: map[string]any{
"seconds": map[string]any{"type": "integer", "description": "how long to sample, in seconds (default 3, at most 10)"},
"limit": map[string]any{"type": "integer", "description": "how many units or containers to name (default 15, at most 50)"},
"by": map[string]any{"type": "string", "enum": []string{"unit", "container"}, "description": "what to rank (default unit)"},
},
Run: func(args map[string]any) (any, error) {
n, err := count(args, "seconds", 3, 1, 10)
if err != nil {
return nil, err
}
w := time.Duration(n) * time.Second
limit, err := count(args, "limit", 15, 1, 50)
if err != nil {
return nil, err
}
by := text(args, "by")
switch by {
case "":
by = "unit"
case "unit", "container":
default:
return nil, fmt.Errorf("by is unit or container, not %q", by)
}
ac, err := parseCPU(m.read("/proc/stat"))
if err != nil {
return nil, err
}
a := m.unitCPU()
m.Wait(w)
bc, err := parseCPU(m.read("/proc/stat"))
if err != nil {
return nil, err
}
b := m.unitCPU()
machine := machineCPUOver(ac, bc)
top, unaccounted := m.topCPU(a, b, w, by, limit, machine.raw)
return map[string]any{
"window_s": w.Seconds(),
"by": by,
"cpu": machine,
"top": top,
"unaccounted_pct": unaccounted,
}, nil
},
},
{ {
Name: "disk_filesystems", Name: "disk_filesystems",
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " + Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
+2 -1
View File
@@ -5,7 +5,8 @@
"disk_load", "disk_load",
"disk_top", "disk_top",
"disk_filesystems", "disk_filesystems",
"disk_devices" "disk_devices",
"cpu_top"
], ],
"build": { "build": {
"artifacts": [ "artifacts": [
+22 -5
View File
@@ -44,7 +44,7 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
| tool | | what | | tool | | what |
|---|---|---| |---|---|---|
| `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) or `patch` (a unified diff, as `git diff` writes it); answers the commit and its parent | | `gitea_commit` | a | one commit on a branch of its own, created from `base` (default the default branch) when absent, carrying either `files` (each `{path, content \| content_base64 \| delete: true}`) `patch` (a unified diff, as `git diff` writes it), or `patch_file` with `patch_sha256` (a patch too large to type, see below); answers the commit and its parent |
| `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject | | `gitea_commits` | r | a ref's history, newest first, optionally only commits touching a path: sha, author, date, subject |
| `gitea_compare` | r | what head holds that base does not: its commits and the files they touch | | `gitea_compare` | r | what head holds that base does not: its commits and the files they touch |
| `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments | | `gitea_pull_reviews` | r | a pull request's reviews — state, the commit reviewed and whether it is the head now, stale, dismissed — with their line comments |
@@ -52,13 +52,28 @@ base64-encoded. The Go bundle `cmd/forge` serves the commit, and the reads an ag
| `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path | | `gitea_search_code` | r | the lines of a ref's text files matching a literal string or a regular expression, optionally under a path |
**Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266): **Where a commit may go** is the rule `gitea_api` already holds for a file written (novox/hq ADR 0266):
never the repository's default branch, never a branch a protection rule names. A rule covers a branch as the never the repository's default branch, never a branch a protection rule names. The tools' account may not read the
rules themselves (the forge keeps them for a repository's administrator, novox/hq ADR 0365): for it an existing branch
is judged by the protection the forge says it applies there, and a branch not made yet by the forge itself, which
refuses the commit that would make a branch a rule guards. A rule covers a branch as the
forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob
(`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob (`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob
the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the
commit. Nothing about merging changes: a merge still needs an independent review at the head and both commit. Nothing about merging changes: a merge still needs an independent review at the head and both
required checks green, through `gitea_merge_pull_request`. required checks green, through `gitea_merge_pull_request`.
**A patch too large to type** (novox/hq issue 852: a feature's diff is megabytes) goes as `patch_file`, with
`patch_sha256`. The caller writes the patch into the module's spool directory `forge-patches`
(`/var/lib/mesh/forge-patches`, mode 0733 and owned by the machine's account, which the tools run as: anyone writes,
nobody lists; the bundle is given its path as `MESH_GITEA_PATCH_DIR`) and names it: `patch_file` is the file's NAME,
32 to 64 characters of `0-9a-f` that the caller chooses at random, never a path. The tool then reads that one file,
applies it exactly as it applies `patch` (same size limit, same answer) and removes it once it has found it, whether
the commit was made or not. A file refused before that (a bad name, a missing hash or spool directory, a name with no
file) stays in the spool. It refuses, sending nothing, a name of another shape, a file that is missing, empty, over
4 MiB, with more than one hard link or not a regular file (a symlink is never followed), and a file whose SHA-256 is not `patch_sha256`: the directory has no sticky bit
(the manifest allows none), so another writer could replace a file, and the hash is how the caller knows the tool read
the bytes it wrote. It reads nothing outside the directory.
**How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the **How the commit is made.** `files` become one call to the forge's multi-file contents endpoint: a path the
ref holds is updated, one it does not is created, and an update or delete carries the blob read just ref holds is updated, one it does not is created, and an update or delete carries the blob read just
before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at before, so a file someone changed meanwhile is refused by the forge rather than overwritten. The blobs are read at
@@ -66,7 +81,7 @@ the commit the branch (or, for a new branch, its base) was read at, never at its
forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the forge's diff endpoint, which applies it; for a small change to a large file this is far shorter than the
file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the file. Neither endpoint takes a commit to build on, only a branch (Gitea 1.27), so the parent answered is the
one the forge says it made the commit on; when the branch moved between the read and the commit, the answer one the forge says it made the commit on; when the branch moved between the read and the commit, the answer
says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the forge's admin account, so its sha is not the says so. One commit carries at most 200 files, 4 MiB per file and 16 MiB together, or a patch of 4 MiB. Either way the commit is the forge's own, made as the tools' account, so its sha is not the
local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a local commit's: the merge check judges the head on the forge. After the write the branch is read back, and a
branch not at the commit the forge named is a failure, never a success. branch not at the commit the forge named is a failure, never a success.
@@ -75,8 +90,10 @@ is read once and searched in memory; a binary file is passed over, a file over 4
`skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A `skipped`, and at most `max` matches (default 100) are answered, with `truncated` when more were found. A
search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`. search that reaches the call's deadline stops there and answers what it found, with `stopped_at_deadline`.
It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), and one It reaches the forge through the same HTTP client as the npm-registry bundle (`internal/forgehttp`), as the
tool call takes at most 50 seconds. tools' own account (novox/hq ADR 0365 §2, issue 469): the token the TypeScript bundle mints for `mesh-tools` and
keeps (`MESH_GITEA_TOKEN_FILE`), sent only when its scopes line names that account, never the admin login. One tool
call takes at most 50 seconds.
`go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on `go test ./cmd/forge` runs it against a fake forge: the files of one commit created, updated and deleted on
a new branch from its base; an existing branch committed on; the default branch, a protected branch and a new branch from its base; an existing branch committed on; the default branch, a protected branch and
unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another unreadable rules refused before anything is sent; a patch sent to the forge; a branch read back at another
+60 -12
View File
@@ -8,6 +8,7 @@ import { ConfiguredToken, MintedToken, type TokenSource } from "./token.js";
import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js"; import { protectionBody, type BranchProtection, type ProtectionWanted } from "./protection.js";
import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js"; import { Refused, checkSentPath, pathSegment, pathSegments } from "./guard.js";
import { isCommit } from "./checks.js"; import { isCommit } from "./checks.js";
import { TOOLS_ACCOUNT } from "./tools-account.js";
/** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every /** A repository's path on the forge, its owner and name each one segment (guard.ts pathSegment): uniform for every
* call, whether the names came from a caller or from the forge's own answer. */ * call, whether the names came from a caller or from the forge's own answer. */
@@ -124,10 +125,12 @@ export interface ReadyMark {
/** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which /** **The accounts whose "ready at" comment counts** (novox/hq issue 722): the trusted setting ready-authors, which
* the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file * the mesh writes to the file MESH_GITEA_READY_AUTHORS_FILE names as a line `authors=<account> <account> …`. A file
* that cannot be read leaves the mesh's own forge account alone, said; never an empty list read as everyone. */ * that cannot be read leaves the mesh's own forge accounts alone, said; never an empty list read as everyone: the
* account the tools act as (novox/hq ADR 0365) and the admin account, whose marks were made before it. */
export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER, export function readyAuthors(file = process.env.MESH_GITEA_READY_AUTHORS_FILE, own = process.env.MESH_GITEA_ADMIN_USER,
read: (f: string) => string = (f) => readFileSync(f, "utf8")): string[] { read: (f: string) => string = (f) => readFileSync(f, "utf8"),
const fallback = own ? [own] : []; tools = process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT): string[] {
const fallback = [...new Set([tools, own].filter((a): a is string => !!a))];
if (!file) return fallback; if (!file) return fallback;
try { try {
const line = read(file).split("\n").find((l) => l.startsWith("authors=")); const line = read(file).split("\n").find((l) => l.startsWith("authors="));
@@ -444,13 +447,15 @@ export class GiteaClient {
* fault in the announcer cannot hide itself. Reads only. */ * fault in the announcer cannot hide itself. Reads only. */
async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> { async openPulls(readyAuthors: string[] = []): Promise<OpenPull[]> {
const out: OpenPull[] = []; const out: OpenPull[] = [];
const rules = new Map<string, BranchProtection[] | undefined>(); const rules = new Map<string, BranchProtection | null | undefined>();
for (const found of await this.searchOpenPulls()) { for (const found of await this.searchOpenPulls()) {
const full = `${found.owner}/${found.name}`; const full = `${found.owner}/${found.name}`;
const pull = await this.getPullRequest(found.owner, found.name, found.number); const pull = await this.getPullRequest(found.owner, found.name, found.number);
if (pull.state !== "open" || !pull.head_sha) continue; if (pull.state !== "open" || !pull.head_sha) continue;
if (!rules.has(full)) rules.set(full, await this.branchProtections(found.owner, found.name).catch(() => undefined)); // The base's protection as the forge applies it to that branch (appliedProtection): readable without administration.
const rule = rules.get(full)?.find((p) => (p.rule_name ?? p.branch_name) === (pull.base || "main")); const base = `${full}:${pull.base || "main"}`;
if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined));
const rule = rules.get(base);
const checks = await this.headChecks(found.owner, found.name, pull.head_sha); const checks = await this.headChecks(found.owner, found.name, pull.head_sha);
const labels = found.labels ?? []; const labels = found.labels ?? [];
// The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label. // The ready mark (novox/hq issue 722): its timeline is read, whole, only for a pull request that carries the label.
@@ -673,16 +678,44 @@ export class GiteaClient {
// ---- Branch protection ---- // ---- Branch protection ----
/** A repository's branch protection rules. */ /** A repository's branch protection rules. The forge answers them to a repository's administrator alone, which the
* tools' account is not (novox/hq ADR 0365 §2): for it this is refused (403), and what a branch's protection asks
* is read with appliedProtection. */
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> { async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? []; return (await this.request<BranchProtection[]>(`${repoPath(owner, repo)}/branch_protections`)) ?? [];
} }
/** The rule named for a branch, null when it has none. */ /** The rule named for a branch, null when it has none — from the rules' list, so a repository administrator's. */
async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> { async branchProtection(owner: string, repo: string, rule: string): Promise<BranchProtection | null> {
return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null; return (await this.branchProtections(owner, repo)).find((p) => (p.rule_name ?? p.branch_name) === rule) ?? null;
} }
/**
* The protection that covers an existing branch, as the forge applies it, or null when none does. Read from the
* branch itself, which anyone who reads the code may read — the rules' own list is a repository administrator's
* (novox/hq ADR 0365 §2) — and as the forge matches it, a glob rule included (its name is the rule's). The
* branch's `user_can_push` stands for `enable_push`: whether this account may push there. A branch that does not
* exist is null: nothing covers it yet, and the forge judges a push that makes it.
*/
async appliedProtection(owner: string, repo: string, branch: string): Promise<BranchProtection | null> {
let b: any;
try {
b = await this.request<any>(`${repoPath(owner, repo)}/branches/${pathSegments("branch", branch)}`);
} catch (err) {
if (/: 404 /.test(String((err as Error)?.message))) return null;
throw err;
}
if (!b?.protected) return null;
return {
rule_name: String(b.effective_branch_protection_name || branch),
branch_name: branch,
enable_push: Boolean(b.user_can_push),
enable_status_check: Boolean(b.enable_status_check),
status_check_contexts: Array.isArray(b.status_check_contexts) ? b.status_check_contexts.map(String) : [],
required_approvals: Number(b.required_approvals ?? 0),
};
}
/** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the /** Make a branch require these commit statuses to merge (novox/hq ADR 0237): the rule named for the
* branch is edited — its required statuses replaced by these, everything else it says kept unless * branch is edited — its required statuses replaced by these, everything else it says kept unless
* asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */ * asked — or created, refusing direct pushes unless asked otherwise. Answers the rule as it is now. */
@@ -925,6 +958,15 @@ export class GiteaAdmin {
throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`); throw new Error(`Gitea admin ${path}: ${res.status} ${detail}`);
} }
/** A repository's branch protection rules, read with the administrator account — the forge answers them to a
* repository's administrator alone, which the tools' account is not (novox/hq ADR 0365 §2, ADR 0375). Only a read. */
async branchProtections(owner: string, repo: string): Promise<BranchProtection[]> {
const path = `${repoPath(owner, repo)}/branch_protections`;
const res = await this.request(path);
if (res.status !== 200) GiteaAdmin.fail(path, res);
return Array.isArray(res.body) ? res.body : [];
}
/** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent /** Ensure the npm-owner org exists. 201 created, 2xx/404-then-created, and 422/409 (a concurrent
* create won the race) are all success. */ * create won the race) are all success. */
async ensureOrg(name: string): Promise<void> { async ensureOrg(name: string): Promise<void> {
@@ -948,12 +990,16 @@ export class GiteaAdmin {
// unit the team does not have — so code read must be said here: without it gitea answers a // unit the team does not have — so code read must be said here: without it gitea answers a
// member's clone of a private repository with "not found", which is how the builder's first // member's clone of a private repository with "not found", which is how the builder's first
// credentialed clone failed against a team that named only packages. // credentialed clone failed against a team that named only packages.
const units = { return this.ensureTeamWith(org, team, {
permission: "read", permission: "read",
units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" }, units_map: { "repo.code": "read", "repo.packages": packageWrite ? "write" : "read" },
includes_all_repositories: true, includes_all_repositories: true,
can_create_org_repo: false, can_create_org_repo: false,
}; });
}
/** Ensure a team exists with exactly these units, and return its id (ensureTeam; the tools' team, tools-account.ts). */
async ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number> {
const found = await this.findTeam(org, team); const found = await this.findTeam(org, team);
if (found !== null) { if (found !== null) {
const patch = await this.request(`/teams/${found}`, { const patch = await this.request(`/teams/${found}`, {
@@ -988,7 +1034,7 @@ export class GiteaAdmin {
* The edit path is taken only when the user actually exists. A 422 from the create is also what * The edit path is taken only when the user actually exists. A 422 from the create is also what
* a plain validation failure returns, and reading it as "already there" made the follow-up edit * a plain validation failure returns, and reading it as "already there" made the follow-up edit
* 404 — burying the create's own message, which is the one that says what is actually wrong. */ * 404 — burying the create's own message, which is the one that says what is actually wrong. */
async ensureUser(username: string, password: string, email: string): Promise<void> { async ensureUser(username: string, password: string, email: string, plain = false): Promise<void> {
const res = await this.request("/admin/users", { const res = await this.request("/admin/users", {
method: "POST", method: "POST",
body: JSON.stringify({ username, email, password, must_change_password: false }), body: JSON.stringify({ username, email, password, must_change_password: false }),
@@ -1002,7 +1048,9 @@ export class GiteaAdmin {
// login_name is required by the admin edit endpoint; for a local user it is the username. // login_name is required by the admin edit endpoint; for a local user it is the username.
// active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong // active and prohibit_login: a deactivated or login-prohibited user is refused like a wrong
// password, so the provisioner's check reports it lost; applying again must undo both. // password, so the provisioner's check reports it lost; applying again must undo both.
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false }), // plain: never a site administrator — one made so by hand is made a plain account again (novox/hq ADR 0365).
body: JSON.stringify({ login_name: username, password, must_change_password: false, active: true, prohibit_login: false,
...(plain ? { admin: false } : {}) }),
}); });
if (patch.status === 200) return; if (patch.status === 200) return;
GiteaAdmin.fail(`/admin/users/${username}`, patch); GiteaAdmin.fail(`/admin/users/${username}`, patch);
+120
View File
@@ -0,0 +1,120 @@
package main
// The commit and the reads act as the tools' own account (novox/hq ADR 0365 §2, issue 469): the token the module
// keeps for `mesh-tools`, never the admin login. That account is no repository's administrator, so the forge
// refuses it the protection rules (403, as gitea's reqAdmin does); the commit then judges an existing branch by the
// protection the forge says it applies, and leaves a branch not made yet to the forge's own judgement.
import (
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"gitea/internal/forgehttp"
)
const toolsToken = "tools-token-never-shown"
// asTools serves the fake forge to a bundle holding the tools' token: only that token is taken, the rules are
// refused as to a non-administrator, and the branches named protected say so as the forge does.
func asTools(t *testing.T, f *fakeForge, protected map[string]string, scopes string) (*Forge, *[]string) {
dir := t.TempDir()
file := filepath.Join(dir, "token")
if err := os.WriteFile(file, []byte(toolsToken+"\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(file+".scopes", []byte(scopes+"\n"), 0o600); err != nil {
t.Fatal(err)
}
var seen []string
const repo = "/api/v1/repos/acme/app"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
seen = append(seen, r.Method+" "+r.URL.Path)
if _, _, basic := r.BasicAuth(); basic || r.Header.Get("Authorization") != "token "+toolsToken {
http.Error(w, "unauthorized", 401)
return
}
if r.URL.Path == repo+"/branch_protections" {
http.Error(w, `{"message":"token does not have the administration of this repository"}`, 403)
return
}
if name := strings.TrimPrefix(r.URL.Path, repo+"/branches/"); r.Method == "GET" && name != r.URL.Path {
if rule, ok := protected[name]; ok {
_, _ = w.Write([]byte(`{"name":"` + name + `","commit":{"id":"c0"},"protected":true,"effective_branch_protection_name":"` + rule + `"}`))
return
}
}
r.SetBasicAuth("mesh-admin", f.password) // the fake's own door, past the token checked above
f.handle(w, r)
}))
t.Cleanup(srv.Close)
return &Forge{Client: forgehttp.Client{URL: srv.URL, TokenFile: file, Account: "mesh-tools"}}, &seen
}
func TestTheToolsAccountCommitsOnANewBranchWithItsTokenThoughTheRulesAreNotItsToRead(t *testing.T) {
f := newFake(t)
forge, _ := asTools(t, f, nil, "read:user,write:issue,write:repository for mesh-tools")
got, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
if err != nil {
t.Fatal(err)
}
if len(f.writes) != 1 || !got.Created {
t.Fatalf("%d writes, %+v", len(f.writes), got)
}
}
func TestTheToolsAccountIsRefusedAProtectedBranchTheForgeSaysItGuardsAndTheDefaultBranch(t *testing.T) {
for branch, rule := range map[string]string{"release/1.0": "release/*", "fix/1-old": "fix/1-old", "main": ""} {
f := newFake(t)
protected := map[string]string{}
if rule != "" {
protected[branch] = rule
}
forge, _ := asTools(t, f, protected, "read:user,write:issue,write:repository for mesh-tools")
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": branch,
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
if err == nil || !strings.Contains(err.Error(), "refused") || !strings.Contains(err.Error(), "ADR 0266") {
t.Errorf("%s: %v", branch, err)
}
if rule != "" && !strings.Contains(err.Error(), rule) {
t.Errorf("%s: the rule %s is not named: %v", branch, rule, err)
}
if len(f.writes)+len(f.patches) != 0 {
t.Errorf("%s: something was written", branch)
}
}
}
func TestATokenKeptForTheAdministratorIsNeverSent(t *testing.T) {
f := newFake(t)
// What the module kept before ADR 0365: the admin's token, its scopes line naming no account.
forge, seen := asTools(t, f, nil, "read:user,write:issue,write:repository")
_, err := commit(context.Background(), forge, ask(t, map[string]any{"branch": "fix/365",
"files": filesArg(map[string]any{"path": "x", "content": "x"})}))
if err == nil || !strings.Contains(err.Error(), "not mesh-tools's") {
t.Fatalf("%v", err)
}
if len(*seen) != 0 {
t.Fatalf("the forge was asked %v", *seen)
}
}
func TestTheBundleHandedATokenFileNeedsNoAdminLogin(t *testing.T) {
t.Setenv("MESH_GITEA_URL", "http://127.0.0.1:3000")
t.Setenv("MESH_GITEA_TOKEN_FILE", "/state/token")
t.Setenv("MESH_GITEA_TOOLS_USER", "")
t.Setenv("MESH_GITEA_ADMIN_USER", "")
t.Setenv("MESH_GITEA_ADMIN_PASSWORD_FILE", "")
f, err := ForgeFromEnv()
if err != nil {
t.Fatal(err)
}
if f.TokenFile != "/state/token" || f.Account != "mesh-tools" || f.User != "" || f.PasswordFile != "" {
t.Fatalf("%+v", f.Client)
}
}
+57 -8
View File
@@ -9,6 +9,8 @@ import (
"net/url" "net/url"
"strings" "strings"
"unicode/utf8" "unicode/utf8"
"gitea/internal/forgehttp"
) )
// gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull // gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull
@@ -33,6 +35,11 @@ type CommitAsk struct {
Message string Message string
Files []FileChange Files []FileChange
Patch string // a unified diff, as git diff writes it, instead of Files Patch string // a unified diff, as git diff writes it, instead of Files
// PatchFile is the name of a file in the spool directory holding the patch (novox/hq issue 852), with the
// SHA-256 its caller says it has; commitAsk reads it into Patch.
PatchFile string
PatchSHA256 string
patchFileGiven bool
} }
// Committed is what gitea_commit made. // Committed is what gitea_commit made.
@@ -67,6 +74,11 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"), a := CommitAsk{Owner: text(args, "owner"), Repo: text(args, "repo"), Branch: text(args, "branch"),
Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))} Base: text(args, "base"), Message: strings.TrimSpace(raw(args, "message"))}
a.Patch = raw(args, "patch") a.Patch = raw(args, "patch")
if v, ok := args["patch_file"]; ok && v != nil {
a.patchFileGiven = true
a.PatchFile = raw(args, "patch_file")
}
a.PatchSHA256 = strings.ToLower(strings.TrimSpace(raw(args, "patch_sha256")))
if a.Branch == "" { if a.Branch == "" {
return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent") return a, refuse("branch is required: the branch the commit goes on, a branch of its own (created from base when absent). Nothing was sent")
} }
@@ -109,11 +121,23 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
a.Files = append(a.Files, fc) a.Files = append(a.Files, fc)
} }
} }
sources := 0
for _, g := range []bool{len(a.Files) > 0, strings.TrimSpace(a.Patch) != "", a.patchFileGiven} {
if g {
sources++
}
}
switch { switch {
case len(a.Files) == 0 && strings.TrimSpace(a.Patch) == "": case sources == 0:
return a, refuse("give files (each {path, content | content_base64 | delete: true}) or patch (a unified diff, as git diff writes it). Nothing was sent") return a, refuse("give files (each {path, content | content_base64 | delete: true}), patch (a unified diff, as git diff writes it) or patch_file (the name of a patch in the spool directory). Nothing was sent")
case len(a.Files) > 0 && strings.TrimSpace(a.Patch) != "": case sources > 1:
return a, refuse("give files or patch, not both: one commit carries one of them. Nothing was sent") return a, refuse("give files, patch or patch_file, not more than one: one commit carries one of them. Nothing was sent")
case a.PatchSHA256 != "" && !a.patchFileGiven:
return a, refuse("patch_sha256 goes with patch_file only. Nothing was sent")
case a.patchFileGiven:
if err := checkPatchFileAsk(a); err != nil {
return a, err
}
case len(a.Files) > maxFiles: case len(a.Files) > maxFiles:
return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles) return a, refuse("%d files is more than the %d one commit carries. Nothing was sent", len(a.Files), maxFiles)
case len(a.Patch) > maxPatchBytes: case len(a.Patch) > maxPatchBytes:
@@ -132,13 +156,13 @@ func readCommitAsk(args map[string]any) (CommitAsk, error) {
return a, refuse("%s is named twice in files. Nothing was sent", fc.Path) return a, refuse("%s is named twice in files. Nothing was sent", fc.Path)
} }
seen[fc.Path] = true seen[fc.Path] = true
given := 0 have := 0
for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} { for _, g := range []bool{fc.Content != nil, fc.ContentBase64 != nil, fc.Delete} {
if g { if g {
given++ have++
} }
} }
if given != 1 { if have != 1 {
return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path) return a, refuse("files[%d] (%s) needs exactly one of content, content_base64 or delete: true. Nothing was sent", i, fc.Path)
} }
if fc.Content != nil && !utf8.ValidString(*fc.Content) { if fc.Content != nil && !utf8.ValidString(*fc.Content) {
@@ -211,7 +235,32 @@ func commit(ctx context.Context, f *Forge, a CommitAsk) (*Committed, error) {
// Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed). // Read the rules, or refuse: a branch the rules could not be read for is never written (fails closed).
var rules []Protection var rules []Protection
if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil { if err := f.GetJSON(ctx, repo+"/branch_protections", &rules); err != nil {
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err) var status *forgehttp.StatusError
if !errors.As(err, &status) || status.Status != http.StatusForbidden {
return nil, fmt.Errorf("the protection rules of %s/%s could not be read, so the commit was not judged and nothing was sent: %w", a.Owner, a.Repo, err)
}
// The rules are a repository administrator's to read, and the tools' account is none (novox/hq ADR 0365
// §2). What it may read is the protection the forge applies to an existing branch: one protected is
// refused here. A branch not made yet is covered by no rule this account can see, and the forge itself
// judges the commit that makes it against every rule, refusing one a rule guards.
if err := targetRefused(a.Branch, r.DefaultBranch, nil); err != nil {
return nil, err
}
b, err := f.branch(ctx, repo, a.Branch)
if err != nil {
return nil, fmt.Errorf("whether a protection rule guards %s of %s/%s could not be read, so nothing was sent: %w", a.Branch, a.Owner, a.Repo, err)
}
if b != nil && b.Protected {
rule := b.Rule
if rule == "" {
rule = a.Branch
}
rules = []Protection{{RuleName: rule}}
if !ruleCovers(rule, a.Branch) {
return nil, refuse("a commit on %s is refused: the protection rule %s guards it (novox/hq ADR 0266). Put it on a "+
"branch of its own. Nothing was sent", a.Branch, rule)
}
}
} }
if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil { if err := targetRefused(a.Branch, r.DefaultBranch, rules); err != nil {
return nil, err return nil, err
+6 -3
View File
@@ -11,8 +11,8 @@ import (
"gitea/internal/forgehttp" "gitea/internal/forgehttp"
) )
// The forge as this bundle reaches it: its own HTTP interface on the machine, as the forge's admin account // The forge as this bundle reaches it: its own HTTP interface on the machine, as the tools' own account (novox/hq
// (internal/forgehttp, shared with the npm-registry bundle beside it). // ADR 0365 §2; internal/forgehttp, shared with the npm-registry bundle beside it).
// Forge is the forge's HTTP interface. // Forge is the forge's HTTP interface.
type Forge struct { type Forge struct {
@@ -91,12 +91,15 @@ func (p Protection) name() string {
return p.BranchName return p.BranchName
} }
// Branch is one branch with the commit it points at. // Branch is one branch with the commit it points at, and the protection the forge applies to it: what an account
// that may not read the rules themselves (the tools' account, novox/hq ADR 0365) knows of them.
type Branch struct { type Branch struct {
Name string `json:"name"` Name string `json:"name"`
Commit struct { Commit struct {
ID string `json:"id"` ID string `json:"id"`
} `json:"commit"` } `json:"commit"`
Protected bool `json:"protected"`
Rule string `json:"effective_branch_protection_name"`
} }
// branch is the named branch, nil when the forge has none of that name. // branch is the named branch, nil when the forge has none of that name.
+7 -1
View File
@@ -619,11 +619,17 @@ func TestToolsAreTheOnesTheReadmeNamesAndTheManifestBuilds(t *testing.T) {
if len(a.Loads) != 1 || a.Loads[0] != a.Binary { if len(a.Loads) != 1 || a.Loads[0] != a.Binary {
t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary) t.Errorf("the bundle loads %v, not its binary %s", a.Loads, a.Binary)
} }
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} { // The tools' own account, never the admin login (novox/hq ADR 0365 §2).
for _, k := range []string{"MESH_GITEA_URL", "MESH_GITEA_TOOLS_USER", "MESH_GITEA_TOKEN_FILE"} {
if a.Env[k] == "" { if a.Env[k] == "" {
t.Errorf("the bundle is not given %s", k) t.Errorf("the bundle is not given %s", k)
} }
} }
for _, k := range []string{"MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD_FILE"} {
if a.Env[k] != "" {
t.Errorf("the bundle is given %s: it acts as the tools' account, not the administrator", k)
}
}
} }
} }
if !found { if !found {
+7 -3
View File
@@ -55,7 +55,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
"the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " + "the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " +
"At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " + "At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " +
"the parent the forge made it on, read back from the branch, and says when the branch moved under " + "the parent the forge made it on, read back from the branch, and says when the branch moved under " +
"it. Replaces git commit and git " + "it. A patch too large to type goes as patch_file with patch_sha256: the name of a file in the " +
"spool directory (read once, applied as patch is, removed afterwards, never read from anywhere " +
"else). Replaces git commit and git " +
"push of a feature branch. (a)", "push of a feature branch. (a)",
Input: map[string]any{ Input: map[string]any{
"type": "object", "type": "object",
@@ -79,7 +81,9 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
"required": []string{"path"}, "required": []string{"path"},
}, },
}, },
"patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this or files"), "patch": str("a unified diff of the change, as git diff or git format-patch writes it; give this, patch_file or files"),
"patch_file": str("for a patch too large to type: the NAME (32 to 64 characters of 0-9a-f, chosen at random by the caller, no path) of a file the caller wrote into the forge patch spool directory (the module's forge-patches directory); needs patch_sha256; give this, patch or files"),
"patch_sha256": str("with patch_file: the file's SHA-256, in hex; the commit is refused, and nothing is sent, when the file's differs"),
}, },
"required": []string{"owner", "repo", "branch", "message"}, "required": []string{"owner", "repo", "branch", "message"},
}, },
@@ -88,7 +92,7 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return commit(ctx, f, a) return commitAsk(ctx, f, os.Getenv(patchDirEnv), a)
}), }),
}, },
{ {
+109
View File
@@ -0,0 +1,109 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"syscall"
)
// A patch too large to type into a call is handed over as a file (novox/hq issue 852). The caller writes it into
// one spool directory the module declares (mode 0733: anyone writes, nobody lists) and names it in patch_file,
// with the file's SHA-256. The tool runs as the machine's account, so it reads nothing but that directory: the
// name is 32 to 64 characters of [0-9a-f], so it holds no separator and no dot; the file must be a regular file
// (never a symlink); and its hash must be the caller's, because without the sticky bit another writer could have
// replaced it. The file is removed once taken, whether the commit was made or not.
const patchDirEnv = "MESH_GITEA_PATCH_DIR"
var (
patchFileName = regexp.MustCompile(`^[0-9a-f]{32,64}$`)
patchSHA = regexp.MustCompile(`^[0-9a-f]{64}$`)
)
// checkPatchFileAsk judges the arguments of a patch_file ask, before any file is looked at.
func checkPatchFileAsk(a CommitAsk) error {
if !patchFileName.MatchString(a.PatchFile) {
return refuse("patch_file %q is not a name: 32 to 64 characters of 0-9a-f, no path (the file's name in the spool directory, chosen at random). Nothing was sent", a.PatchFile)
}
if !patchSHA.MatchString(a.PatchSHA256) {
return refuse("patch_file needs patch_sha256: the file's SHA-256, 64 hex characters. Nothing was sent")
}
return nil
}
// commitAsk makes the commit the ask names; a patch_file is first read from the spool directory into the patch
// and removed afterwards.
func commitAsk(ctx context.Context, f *Forge, spool string, a CommitAsk) (*Committed, error) {
if !a.patchFileGiven {
return commit(ctx, f, a)
}
if err := checkPatchFileAsk(a); err != nil {
return nil, err
}
if spool == "" {
return nil, refuse("patch_file is not available: %s is not set, so there is no spool directory. Nothing was sent", patchDirEnv)
}
path := filepath.Join(spool, a.PatchFile)
defer func() {
// Whatever happened, the file is not left: removing a symlink removes the link, never its target.
if st, err := os.Lstat(path); err == nil && !st.IsDir() {
_ = os.Remove(path)
}
}()
body, err := readSpooled(path, a.PatchFile)
if err != nil {
return nil, err
}
got := sha256.Sum256(body)
if hex.EncodeToString(got[:]) != a.PatchSHA256 {
return nil, refuse("the file %s is not the patch the caller hashed: its SHA-256 is %s, patch_sha256 says %s. Nothing was sent",
a.PatchFile, hex.EncodeToString(got[:]), a.PatchSHA256)
}
a.Patch = string(body)
return commit(ctx, f, a)
}
// readSpooled reads a regular file of at most maxPatchBytes, never through a symlink.
func readSpooled(path, name string) ([]byte, error) {
st, err := os.Lstat(path)
switch {
case errors.Is(err, os.ErrNotExist):
return nil, refuse("patch_file %s: no such file in the spool directory. Nothing was sent", name)
case err != nil:
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
case !st.Mode().IsRegular():
return nil, refuse("patch_file %s is not a regular file (a symlink or a directory is never followed). Nothing was sent", name)
case st.Size() > maxPatchBytes:
return nil, refuse("patch_file %s is %s, more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(int(st.Size())), mib(maxPatchBytes))
}
fh, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
if err != nil {
return nil, refuse("patch_file %s cannot be opened: %v. Nothing was sent", name, err)
}
defer fh.Close()
fst, err := fh.Stat()
if err != nil || !fst.Mode().IsRegular() {
return nil, refuse("patch_file %s is not a regular file. Nothing was sent", name)
}
// A hard link to a file elsewhere is no symlink, but it is not the caller's own file either.
if sys, ok := fst.Sys().(*syscall.Stat_t); !ok || sys.Nlink > 1 {
return nil, refuse("patch_file %s has more than one hard link: it must be a file of its own. Nothing was sent", name)
}
body, err := io.ReadAll(io.LimitReader(fh, maxPatchBytes+1))
switch {
case err != nil:
return nil, refuse("patch_file %s cannot be read: %v. Nothing was sent", name, err)
case len(body) > maxPatchBytes:
return nil, refuse("patch_file %s is more than the %s one commit carries: split it into several commits. Nothing was sent", name, mib(maxPatchBytes))
case strings.TrimSpace(string(body)) == "":
return nil, refuse("patch_file %s is empty. Nothing was sent", name)
}
return body, nil
}
+183
View File
@@ -0,0 +1,183 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
)
// novox/hq issue 852: a patch too large to type into a call is handed over as a file in one spool directory.
const spoolName = "0123456789abcdef0123456789abcdef"
const spoolPatch = "diff --git a/README.md b/README.md\n--- a/README.md\n+++ b/README.md\n@@ -1 +1 @@\n-hello\n+hi\n"
func sum(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
func spoolWith(t *testing.T, name, content string) string {
t.Helper()
dir := t.TempDir()
if name != "" {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil {
t.Fatal(err)
}
}
return dir
}
func fileAsk(t *testing.T, name, sha string) CommitAsk {
t.Helper()
return ask(t, map[string]any{"branch": "fix/9", "patch_file": name, "patch_sha256": sha})
}
func TestAPatchFileIsAppliedAsThePatchWouldBeAndTheFileIsGone(t *testing.T) {
f := newFake(t)
dir := spoolWith(t, spoolName, spoolPatch)
got, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, spoolName, sum(spoolPatch)))
if err != nil {
t.Fatal(err)
}
if len(f.patches) != 1 || f.patches[0]["content"] != spoolPatch || f.patches[0]["new_branch"] != "fix/9" || got.Commit != "p1" {
t.Fatalf("%v %+v", f.patches, got)
}
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
t.Fatalf("the file is still there: %v", err)
}
}
func TestAPatchFileIsRemovedWhenTheForgeRefusesIt(t *testing.T) {
f := newFake(t)
dir := spoolWith(t, spoolName, spoolPatch)
// the default branch is refused after the file was read
a := ask(t, map[string]any{"branch": "main", "patch_file": spoolName, "patch_sha256": sum(spoolPatch)})
if _, err := commitAsk(context.Background(), f.serve(), dir, a); err == nil {
t.Fatal("taken")
}
if _, err := os.Lstat(filepath.Join(dir, spoolName)); !os.IsNotExist(err) {
t.Fatalf("the file is still there: %v", err)
}
}
func TestAPatchFileThatCannotBeTakenIsRefusedBeforeAnythingIsSent(t *testing.T) {
outside := filepath.Join(t.TempDir(), "secret")
if err := os.WriteFile(outside, []byte(spoolPatch), 0o600); err != nil {
t.Fatal(err)
}
type tc struct {
name string
setup func(dir string)
file string
sha string
want string
}
for _, c := range []tc{
{"wrong sha", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), []byte(spoolPatch), 0o600) }, spoolName, sum("other"), "sha256"},
{"not found", func(string) {}, spoolName, sum(spoolPatch), "no such file"},
{"symlink", func(d string) { _ = os.Symlink(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "regular file"},
{"hard link", func(d string) { _ = os.Link(outside, filepath.Join(d, spoolName)) }, spoolName, sum(spoolPatch), "hard link"},
{"directory", func(d string) { _ = os.Mkdir(filepath.Join(d, spoolName), 0o700) }, spoolName, sum(spoolPatch), "regular file"},
{"too large", func(d string) {
_ = os.WriteFile(filepath.Join(d, spoolName), []byte(strings.Repeat("x", maxPatchBytes+1)), 0o600)
}, spoolName, sum(spoolPatch), "MiB"},
{"empty", func(d string) { _ = os.WriteFile(filepath.Join(d, spoolName), nil, 0o600) }, spoolName, sum(""), "empty"},
} {
f := newFake(t)
dir := t.TempDir()
c.setup(dir)
_, err := commitAsk(context.Background(), f.serve(), dir, fileAsk(t, c.file, c.sha))
if err == nil || !strings.Contains(err.Error(), c.want) || !strings.Contains(err.Error(), "Nothing was sent") {
t.Errorf("%s: %v", c.name, err)
}
if len(f.patches)+len(f.writes) != 0 {
t.Errorf("%s: something was sent", c.name)
}
}
if b, err := os.ReadFile(outside); err != nil || string(b) != spoolPatch {
t.Errorf("the file outside the spool was touched: %v", err)
}
}
func TestAPatchFileAskThatCannotBeReadIsRefused(t *testing.T) {
ok := sum(spoolPatch)
for name, args := range map[string]map[string]any{
"no sha": {"patch_file": spoolName},
"sha not hex": {"patch_file": spoolName, "patch_sha256": strings.Repeat("z", 64)},
"sha too short": {"patch_file": spoolName, "patch_sha256": ok[:63]},
"empty name": {"patch_file": "", "patch_sha256": ok, "files": filesArg()},
"dotdot": {"patch_file": "../" + spoolName, "patch_sha256": ok},
"slash": {"patch_file": "a/" + spoolName, "patch_sha256": ok},
"absolute": {"patch_file": "/etc/passwd", "patch_sha256": ok},
"too short": {"patch_file": strings.Repeat("a", 31), "patch_sha256": ok},
"too long": {"patch_file": strings.Repeat("a", 65), "patch_sha256": ok},
"upper case": {"patch_file": strings.ToUpper(spoolName), "patch_sha256": ok},
"not hex": {"patch_file": strings.Repeat("g", 32), "patch_sha256": ok},
"with patch": {"patch_file": spoolName, "patch_sha256": ok, "patch": "diff"},
"with files": {"patch_file": spoolName, "patch_sha256": ok, "files": filesArg(map[string]any{"path": "x", "content": "x"})},
"sha without it": {"patch": "diff", "patch_sha256": ok},
} {
full := map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b"}
for k, v := range args {
full[k] = v
}
_, err := readCommitAsk(full)
if err == nil || !strings.Contains(err.Error(), "Nothing was sent") {
t.Errorf("%s: %v", name, err)
}
}
// 64 characters is taken; so is an upper-case sha.
for _, n := range []string{spoolName, strings.Repeat("ab", 32)} {
if _, err := readCommitAsk(map[string]any{"owner": "acme", "repo": "app", "message": "m", "branch": "b",
"patch_file": n, "patch_sha256": strings.ToUpper(sum(spoolPatch))}); err != nil {
t.Errorf("%s: %v", n, err)
}
}
}
func TestAPatchFileWithoutASpoolDirectoryIsRefused(t *testing.T) {
f := newFake(t)
_, err := commitAsk(context.Background(), f.serve(), "", fileAsk(t, spoolName, sum(spoolPatch)))
if err == nil || !strings.Contains(err.Error(), "MESH_GITEA_PATCH_DIR") || !strings.Contains(err.Error(), "Nothing was sent") {
t.Fatalf("%v", err)
}
}
func TestTheManifestDeclaresTheSpoolDirectoryAndGivesItToTheBundle(t *testing.T) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m struct {
Resources []struct {
ID, Type, Path, Mode, Owner string
} `json:"resources"`
Build struct {
Artifacts []struct {
From string `json:"from"`
Env map[string]string `json:"env"`
} `json:"artifacts"`
} `json:"build"`
}
if err := jsonInto(raw, &m); err != nil {
t.Fatal(err)
}
found := false
for _, r := range m.Resources {
if r.ID == "forge-patches" {
found = true
if r.Type != "directory" || r.Mode != "0733" || r.Owner != "${machine:account}" || r.Path == "" {
t.Errorf("%+v", r)
}
}
}
if !found {
t.Error("no forge-patches directory")
}
for _, a := range m.Build.Artifacts {
if a.From == "cmd/forge" && a.Env["MESH_GITEA_PATCH_DIR"] != "${dir:forge-patches}" {
t.Errorf("env %v", a.Env)
}
}
}
+24 -28
View File
@@ -42,9 +42,9 @@ import { commitStatusTool } from "./delivery.js";
import { mergeTool } from "./merge.js"; import { mergeTool } from "./merge.js";
import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { GiteaAdmin, GiteaClient, movedSince } from "./client.js";
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked, import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, headsToAnnounce, statusesFor, type Announced, type Checked,
type RepoCheckFacts, reposToLook } from "./pulls.js"; repoCheckFacts, reposToLook } from "./pulls.js";
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js"; import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, keptSecret, receiver, type ForgeEvent } from "./webhook.js"; import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js"; import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is // Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
@@ -400,22 +400,6 @@ async function setVerdict(client: GiteaClient, event: { body: unknown }): Promis
console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`); console.log(`[gitea] ${c.owner}/${c.repo}#${c.number ?? "?"} at ${c.commit.slice(0, 8)}: merge check ${c.verdict}`);
} }
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
// required would let an untested repository merge.
async function repoCheckFacts(client: GiteaClient, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
if (c["repo-check"]?.verdict !== "warning") return {};
const defined = await client.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
if (defined !== false) return { defined };
const rules = await client.branchProtections(c.owner, c.repo).catch(() => undefined);
if (!rules) return { defined };
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === (base || "main"));
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
return { defined, required };
}
// **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words, // **This module is the one writer of a head's mesh/merge-gate** (novox/hq issue 446): the announcement's pending words,
// where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last // where the check is when a build machine takes it, and the verdict — each head's writes in one chain, the verdict last
// (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending. // (gate.ts). A second writer elsewhere raced the verdict and could leave a pass reading pending.
@@ -429,10 +413,16 @@ const gates = new GateWriter({
// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own // **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own
// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to // process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to
// reach, nothing is served, as the tools' bundle serves nothing then. // reach, nothing is served, as the tools' bundle serves nothing then.
registerModuleTools("gitea", () => (gitea ? [commitStatusTool({ /** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what
* gitea_events says of them (webhook.ts). */
const pace = new Pace();
if (!gitea) pace.unheard("the forge is not watched: no way to a token", false);
// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729).
registerModuleTools("gitea", () => [eventsTool(pace), ...(gitea ? [commitStatusTool({
setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status),
gates, gates,
}), mergeTool(gitea, { gates })] : [])); }), mergeTool(gitea, { gates })] : [])]);
// **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when // **A build machine took an ask** (the build seat's `started`, novox/hq ADR 0157): said on the head it checks, when
// the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing. // the head is an open pull request's this module announced. Any other build — a branch, a merge's — says nothing.
@@ -451,9 +441,6 @@ async function buildStarted(event: { body: unknown }): Promise<void> {
} }
} }
/** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are (webhook.ts). */
const pace = new Pace();
/** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */ /** What the forge's event asks of the announcer at once (issue 729): a new head is announced, and a merge. */
async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> { async function actOn(client: GiteaClient, event: ForgeEvent): Promise<void> {
const b = event.body as { owner?: string; repo?: string; number?: number }; const b = event.body as { owner?: string; repo?: string; number?: number };
@@ -476,23 +463,32 @@ async function listenForForge(client: GiteaClient): Promise<boolean> {
const url = process.env.MESH_GITEA_HOOK_URL; const url = process.env.MESH_GITEA_HOOK_URL;
if (!stateDir || !url) { if (!stateDir || !url) {
console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge"); console.log("[gitea] no webhook receiver: MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset, so the polls alone watch the forge");
pace.unheard("MESH_GITEA_STATE_DIR or MESH_GITEA_HOOK_URL unset", false);
return false; return false;
} }
try { try {
const secret = keptSecret(join(stateDir, "webhook.secret")); const secret = keptSecret(join(stateDir, "webhook.secret"));
const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT); const port = Number(process.env.MESH_GITEA_HOOK_PORT ?? HOOK_PORT);
const server = receiver({ secret, port, heard: () => pace.heard(), const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0";
const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery),
refused: (reason, from) => pace.refused(reason, from),
publish: (event) => emit(event.name, event.body), publish: (event) => emit(event.name, event.body),
act: (event) => actOn(client, event) }); act: (event) => actOn(client, event) });
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
server.once("error", reject); server.once("error", reject);
server.listen(port, process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0", () => resolve()); server.listen(port, host, () => resolve());
}); });
const did = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret); const hook = await ensureSystemHook(GiteaAdmin.fromEnv(), url.replace(/\/+$/, "") + HOOK_PATH, secret);
console.log(`[gitea] listening for the forge's events on port ${port}; its system webhook ${did}, every event, signed`); pace.listening(host, port, hook.url, hook);
console.log(`[gitea] listening for the forge's events on ${host}:${port}; its system webhook ${hook.id} ${hook.did} ` +
`at ${hook.url} (in place of ${hook.removed.system} system, ${hook.removed.default} default and ` +
`${hook.removed.repository} repository webhook(s)${hook.removed.repositories.length ? ` in ${hook.removed.repositories.join(", ")}` : ""}), ` +
`${hook.events.length} event families, signed`);
return true; return true;
} catch (err) { } catch (err) {
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${err instanceof Error ? err.message : err}`); const why = err instanceof Error ? err.message : String(err);
pace.unheard(why);
console.error(`[gitea] the forge's events are not heard, so the polls keep their pace — ${why}`);
return false; return false;
} }
} }
+76 -11
View File
@@ -1,7 +1,17 @@
// Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge, // Package forgehttp is the forge's HTTP interface as this module's Go bundles reach it (cmd/forge,
// cmd/npm-registry): the forge on its own machine, as its admin account, whose password the vault delivered // cmd/npm-registry), on the forge's own machine. One copy for both bundles (novox/hq issue 539).
// into a file this module owns (novox/hq ADR 0086), by basic authentication. The password is read when a call //
// is made and never logged, answered or put in an error. One copy for both bundles (novox/hq issue 539). // Two ways in, by what the manifest hands the bundle:
//
// - **as the tools' account** (novox/hq ADR 0365 §2), when MESH_GITEA_TOKEN_FILE names the token the module
// keeps for it (token.ts mints it, for `mesh-tools`, and keeps it at 0600 with its scopes line beside it). A
// token whose scopes line does not say it is the tools' account's — one kept from before, the administrator's —
// is never sent: the call fails, saying so, until the module has minted the account's own. This bundle mints
// nothing; a token the forge rejects is renewed by the module's next call, and said here.
// - **as the admin account**, whose password the vault delivered into a file this module owns (novox/hq ADR
// 0086), by basic authentication, when no token file is named.
//
// A credential is read when a call is made and never logged, answered or put in an error.
package forgehttp package forgehttp
import ( import (
@@ -25,12 +35,31 @@ type Client struct {
URL string // e.g. http://127.0.0.1:3000 URL string // e.g. http://127.0.0.1:3000
User string User string
PasswordFile string PasswordFile string
HTTP *http.Client // when nil, a client with Timeout // TokenFile, when set, is the tools' account's token (MESH_GITEA_TOKEN_FILE), sent instead of the admin login;
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero // Account the account its scopes line must name (MESH_GITEA_TOOLS_USER, default mesh-tools).
TokenFile string
Account string
HTTP *http.Client // when nil, a client with Timeout
Timeout time.Duration // one request's limit when HTTP is nil; 20 s when zero
} }
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. // ToolsAccount is the account the forge's tools act as when the manifest names none (token.ts, tools-account.ts).
const ToolsAccount = "mesh-tools"
// FromEnv is the forge as the manifest hands it to a bundle, and the names of the variables it left unset. A bundle
// handed MESH_GITEA_TOKEN_FILE acts as the tools' account and needs no admin login.
func FromEnv() (Client, []string) { func FromEnv() (Client, []string) {
if file := os.Getenv("MESH_GITEA_TOKEN_FILE"); file != "" {
c := Client{URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), TokenFile: file,
Account: os.Getenv("MESH_GITEA_TOOLS_USER")}
if c.Account == "" {
c.Account = ToolsAccount
}
if c.URL == "" {
return c, []string{"MESH_GITEA_URL"}
}
return c, nil
}
c := Client{ c := Client{
URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"), URL: strings.TrimRight(os.Getenv("MESH_GITEA_URL"), "/"),
User: os.Getenv("MESH_GITEA_ADMIN_USER"), User: os.Getenv("MESH_GITEA_ADMIN_USER"),
@@ -80,14 +109,45 @@ func (c *Client) password() (string, error) {
return strings.TrimSpace(string(raw)), nil return strings.TrimSpace(string(raw)), nil
} }
// token is the tools' account's token, refused unless its scopes line names that account.
func (c *Client) token() (string, error) {
raw, err := os.ReadFile(c.TokenFile)
if err != nil {
return "", fmt.Errorf("the tools' token is not kept yet (the module mints it for %s at its first call): %w", c.Account, err)
}
line, err := os.ReadFile(c.TokenFile + ".scopes")
if err != nil || !strings.HasSuffix(strings.TrimSpace(string(line)), " for "+c.Account) {
return "", fmt.Errorf("the token kept at %s is not %s's (novox/hq ADR 0365): the module mints that account's own at its next call; nothing was sent", c.TokenFile, c.Account)
}
token := strings.TrimSpace(string(raw))
if token == "" {
return "", fmt.Errorf("the token kept at %s is empty; nothing was sent", c.TokenFile)
}
return token, nil
}
// authorize puts the bundle's credential on a request: the tools' token, or the admin login.
func (c *Client) authorize(req *http.Request) error {
if c.TokenFile != "" {
token, err := c.token()
if err != nil {
return err
}
req.Header.Set("Authorization", "token "+token)
return nil
}
password, err := c.password()
if err != nil {
return err
}
req.SetBasicAuth(c.User, password)
return nil
}
// Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit; // Do sends one request, with payload as its JSON body when not nil, and answers the body, read up to limit;
// 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it // 404 is ErrNotFound. One byte past the limit is read, so a body the limit cut is told from one that fits it
// exactly: a cut body is refused, never read as if it were whole. // exactly: a cut body is refused, never read as if it were whole.
func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) { func (c *Client) Do(ctx context.Context, method, path string, payload any, limit int64) ([]byte, error) {
password, err := c.password()
if err != nil {
return nil, err
}
var body io.Reader var body io.Reader
if payload != nil { if payload != nil {
raw, err := json.Marshal(payload) raw, err := json.Marshal(payload)
@@ -100,7 +160,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
if err != nil { if err != nil {
return nil, err return nil, err
} }
req.SetBasicAuth(c.User, password) if err := c.authorize(req); err != nil {
return nil, err
}
req.Header.Set("Accept", "application/json") req.Header.Set("Accept", "application/json")
if payload != nil { if payload != nil {
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
@@ -118,6 +180,9 @@ func (c *Client) Do(ctx context.Context, method, path string, payload any, limit
return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20) return nil, fmt.Errorf("the forge's answer to %s %s is larger than %d MiB and was not read whole", method, path, limit>>20)
} }
switch { switch {
case res.StatusCode == http.StatusUnauthorized && c.TokenFile != "":
return nil, fmt.Errorf("the forge rejected the tools' token for %s (401): the module mints it afresh at its next call; nothing was done: %w",
method+" "+path, &StatusError{method, path, res.StatusCode, "unauthorized"})
case res.StatusCode == http.StatusNotFound: case res.StatusCode == http.StatusNotFound:
return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound) return nil, fmt.Errorf("%s %s: %w", method, path, ErrNotFound)
case res.StatusCode >= 300: case res.StatusCode >= 300:
+5 -4
View File
@@ -54,9 +54,10 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
const method = args.method ? String(args.method) : "merge"; const method = args.method ? String(args.method) : "merge";
if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`);
const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch);
// **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here rather than left // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the
// to the forge, whose administrator — the account this tool's token is minted for — may merge past a // forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge
// rule that lets it. A rule that cannot be read is no rule passed. // applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that
// cannot be read is no rule passed.
// **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d,
// force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the
// head was and nothing told it which head the review had approved. The caller names it; a head that moved // head was and nothing told it which head the review had approved. The caller names it; a head that moved
@@ -80,7 +81,7 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` +
`${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`);
} }
const rule = await gitea.branchProtection(owner, repo, pull.base); const rule = await gitea.appliedProtection(owner, repo, pull.base);
// Only into a branch whose protection requires the checks: a pull request whose base was moved to a // Only into a branch whose protection requires the checks: a pull request whose base was moved to a
// branch nothing guards would merge past every check (novox/hq ADR 0266). // branch nothing guards would merge past every check (novox/hq ADR 0266).
if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) { if (!rule || !rule.enable_status_check || !(rule.status_check_contexts ?? []).filter(Boolean).length) {
+13 -4
View File
@@ -159,8 +159,8 @@
"settings": { "settings": {
"ready-authors": { "ready-authors": {
"kind": "preference", "kind": "preference",
"default": "mesh-admin jschoubben", "default": "mesh-tools mesh-admin jschoubben",
"why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the mesh's own forge account and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word" "why": "the forge accounts whose comment \"ready at <full head sha>\" marks a pull request ready for mesh-delivery to merge, separated by spaces: the account the forge's tools act as (novox/hq ADR 0365), the admin account for marks made before the tools acted as their own, and the operator's (novox/hq issue 722). A comment by any other account is ignored and said in the forge's log. Trusted: who may have a change merged is the operator's word"
} }
}, },
"resources": [ "resources": [
@@ -184,6 +184,13 @@
"trusted": true, "trusted": true,
"content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n" "content": "# The accounts whose \"ready at <sha>\" comment marks a pull request ready (novox/hq issue 722), from the setting\n# ready-authors, written by the mesh. Replaced on every push; change the setting, never this file.\nauthors=${setting:ready-authors}\n"
}, },
{
"id": "forge-patches",
"type": "directory",
"path": "/var/lib/mesh/forge-patches",
"mode": "0733",
"owner": "${machine:account}"
},
{ {
"id": "state", "id": "state",
"type": "directory", "type": "directory",
@@ -308,6 +315,7 @@
"MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json", "MESH_GITEA_CONFIG_FILE": "${dir:mesh-state}/config.json",
"MESH_GITEA_ADMIN_USER": "mesh-admin", "MESH_GITEA_ADMIN_USER": "mesh-admin",
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret", "MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret",
"MESH_GITEA_TOOLS_USER": "mesh-tools",
"MESH_GITEA_STATE_DIR": "${dir:runtime-state}", "MESH_GITEA_STATE_DIR": "${dir:runtime-state}",
"MESH_RECEIVES": "${dir:grants}/npm.json", "MESH_RECEIVES": "${dir:grants}/npm.json",
"MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors", "MESH_GITEA_READY_AUTHORS_FILE": "${dir:mesh-state}/ready-authors",
@@ -345,8 +353,9 @@
], ],
"env": { "env": {
"MESH_GITEA_URL": "http://127.0.0.1:${port:3000}", "MESH_GITEA_URL": "http://127.0.0.1:${port:3000}",
"MESH_GITEA_ADMIN_USER": "mesh-admin", "MESH_GITEA_TOOLS_USER": "mesh-tools",
"MESH_GITEA_ADMIN_PASSWORD_FILE": "${dir:state}/admin.secret" "MESH_GITEA_TOKEN_FILE": "${dir:runtime-state}/token",
"MESH_GITEA_PATCH_DIR": "${dir:forge-patches}"
} }
} }
] ]
+19 -1
View File
@@ -9,7 +9,7 @@
// //
// Pure functions here, so they are tested without a forge; index.ts does the asking and the setting. // Pure functions here, so they are tested without a forge; index.ts does the asking and the setting.
import type { CommitStatus, GiteaPull, GiteaRepo } from "./client.js"; import type { CommitStatus, GiteaClient, GiteaPull, GiteaRepo } from "./client.js";
import { movedSince } from "./client.js"; import { movedSince } from "./client.js";
/** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */ /** The status context a merge check's gate is kept under: one per commit, the newest replacing the last. */
@@ -176,6 +176,24 @@ export interface RepoCheckFacts {
required?: boolean; required?: boolean;
} }
// **What only the forge knows of a repository check said as a warning** (novox/hq issue 293): whether the
// head holds a merge-check.sh at all, and — when it does not — whether the base branch's protection
// requires mesh/repo-check. Asked only for a warning; unknown is left undefined, which statusesFor reads
// as possibly required: a failure on a status nothing requires blocks nothing, a success on one that is
// required would let an untested repository merge. The base's protection is the one the forge applies, read
// from the branch: the rules' own list is a repository administrator's, which the tools' account is not
// (novox/hq ADR 0365).
export async function repoCheckFacts(
forge: Pick<GiteaClient, "holdsFile" | "appliedProtection">, c: Checked, base: string | undefined): Promise<RepoCheckFacts> {
if (c["repo-check"]?.verdict !== "warning") return {};
const defined = await forge.holdsFile(c.owner, c.repo, c.commit, "merge-check.sh").catch(() => undefined);
if (defined !== false) return { defined };
const rule = await forge.appliedProtection(c.owner, c.repo, base || "main").catch(() => undefined);
if (rule === undefined) return { defined };
const required = !!rule?.enable_status_check && (rule.status_check_contexts ?? []).includes(REPO_CHECK_CONTEXT);
return { defined, required };
}
/** The description of a required repository check that is not defined. */ /** The description of a required repository check that is not defined. */
export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " + export const UNDEFINED_REQUIRED = "fail: mesh/repo-check is required here and this head defines no merge-check.sh — " +
"a person adds one, or lifts the requirement from the branch's protection"; "a person adds one, or lifts the requirement from the branch's protection";
+15 -6
View File
@@ -47,11 +47,18 @@ test("every open pull request is read with what its base requires and what its h
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) { if (url.startsWith("/api/v1/repos/novox/mesh-sdk/pulls/14")) {
return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" }); return send({ number: 14, state: "open", base: { ref: "main" }, head: { ref: "y", sha: "bbb2222" }, html_url: "s" });
} }
if (url.startsWith("/api/v1/repos/novox/hq/branch_protections")) { // The base's protection, read from the branch as the tools' account may (novox/hq ADR 0365); the rules' list is refused it.
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/repo-check"] }]); if (url.includes("/branch_protections")) {
res.statusCode = 403;
return res.end(JSON.stringify({ message: "the rules are a repository administrator's" }));
} }
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branch_protections")) { if (url.startsWith("/api/v1/repos/novox/hq/branches/main")) {
return send([{ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] }]); return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
status_check_contexts: ["mesh/repo-check"] });
}
if (url.startsWith("/api/v1/repos/novox/mesh-sdk/branches/main")) {
return send({ name: "main", protected: true, effective_branch_protection_name: "main", enable_status_check: true,
status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"] });
} }
if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] }); if (url.includes("/commits/aaa1111/status")) return send({ statuses: [] });
if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending", if (url.includes("/commits/bbb2222/status")) return send({ statuses: [{ context: "mesh/merge-gate", status: "pending",
@@ -251,6 +258,8 @@ test("a long thread hides nothing: every page of the timeline is read", async ()
test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => { test("the accounts that may mark are the setting's, and only the mesh's own when it cannot be read", async () => {
const { readyAuthors } = await import("../client.ts"); const { readyAuthors } = await import("../client.ts");
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]); assert.deepEqual(readyAuthors("f", "mesh-admin", () => "# x\nauthors=mesh-admin jschoubben\n"), ["mesh-admin", "jschoubben"]);
assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }), ["mesh-admin"]); // The tools' account and the admin's, whose marks were made before ADR 0365 (issue 469).
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n"), ["mesh-admin"]); assert.deepEqual(readyAuthors("f", "mesh-admin", () => { throw new Error("gone"); }, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
assert.deepEqual(readyAuthors("f", "mesh-admin", () => "authors=\n", "mesh-tools"), ["mesh-tools", "mesh-admin"]);
assert.deepEqual(readyAuthors(undefined, "mesh-admin", undefined, "mesh-tools"), ["mesh-tools", "mesh-admin"]);
}); });
+19 -1
View File
@@ -30,6 +30,9 @@ const state = {
// What the tools said on the bus. // What the tools said on the bus.
emitted: [] as { type: string; body: any }[], emitted: [] as { type: string; body: any }[],
rules: [] as any[], rules: [] as any[],
// Whether the token is a repository administrator's. The tools' account is not (novox/hq ADR 0365): the forge
// refuses it the rules' list, as gitea's reqAdmin does, and it reads a branch's protection from the branch.
admin: false,
lastCommit: "2026-01-01T00:00:00Z", lastCommit: "2026-01-01T00:00:00Z",
// The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's // The pull request's head; the head the merge commit joined (the same, unless a test says otherwise) — the forge's
// record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the // record of the pull request keeps saying `head` either way, as it would in the race issue 381 is about; and the
@@ -59,6 +62,7 @@ beforeEach(() => {
state.trunk = TRUNK; state.trunk = TRUNK;
state.emitted = []; state.emitted = [];
state.rules = [protectedMain()]; state.rules = [protectedMain()];
state.admin = false;
state.head = REVIEWED; state.head = REVIEWED;
state.landed = ""; state.landed = "";
state.merged = false; state.merged = false;
@@ -87,8 +91,18 @@ const server = createServer(async (req, res) => {
if (method !== "GET") state.sent.push({ method, path, body }); if (method !== "GET") state.sent.push({ method, path, body });
const repo = "/repos/novox/mesh-catalog"; const repo = "/repos/novox/mesh-catalog";
if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" }); if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: "main" });
if (path.startsWith(`${repo}/branch_protections`) && !state.admin) {
return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
}
if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules); if (method === "GET" && path === `${repo}/branch_protections`) return json(res, 200, state.rules);
if (method === "GET" && path === `${repo}/branches/main`) return json(res, 200, { name: "main", commit: { id: state.trunk } }); const branch = method === "GET" && path.startsWith(`${repo}/branches/`) ? decodeURIComponent(path.slice(`${repo}/branches/`.length)) : null;
if (branch !== null) {
// As the forge says it to any reader of the code: the rule that covers the branch, as the forge matches it.
const rule = (state.rules as any[]).find((r) => ruleCovers(String(r.rule_name ?? r.branch_name), branch));
return json(res, 200, { name: branch, commit: { id: branch === "main" ? state.trunk : "c0ffee" }, protected: !!rule,
...(rule ? { effective_branch_protection_name: rule.rule_name ?? rule.branch_name, enable_status_check: !!rule.enable_status_check,
status_check_contexts: rule.status_check_contexts ?? [], required_approvals: rule.required_approvals ?? 0, user_can_push: false } : {}) });
}
if (method === "GET" && path === `${repo}/pulls/7/files`) { if (method === "GET" && path === `${repo}/pulls/7/files`) {
return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []); return json(res, 200, url.searchParams.get("page") === "1" ? [{ filename: "modules/gitea/x.ts", status: "modified" }] : []);
} }
@@ -199,6 +213,7 @@ test("a protection rule covers a branch as the forge decides it: classes, altern
}); });
test("gitea_api still writes what an author writes", async () => { test("gitea_api still writes what an author writes", async () => {
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/issues/3/comments", method: "POST", body: { body: "hi" } });
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/pulls/7/reviews", method: "POST", body: { event: "COMMENT" } });
await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" }); await tool("gitea_api").run({ path: "/repos/novox/mesh-catalog/branch_protections" });
@@ -206,6 +221,7 @@ test("gitea_api still writes what an author writes", async () => {
}); });
test("gitea_branch_protection_set only strengthens", async () => { test("gitea_branch_protection_set only strengthens", async () => {
state.admin = true; // a repository administrator's token: the tools' account is refused the rules by the forge
const set = (args: Record<string, unknown>) => const set = (args: Record<string, unknown>) =>
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args }); tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-catalog", branch: "main", ...args });
await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/); await assert.rejects(set({ status_checks: "" }), /drops the required status mesh\/merge-gate, mesh\/repo-check/);
@@ -396,6 +412,7 @@ test("gitea_merge_pull_request reads back a fast-forward, and says it cannot rea
}); });
test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => { test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => {
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm }); const del = (owner: string, name: string, confirm: unknown) => tool("gitea_delete_repo").run({ owner, name, confirm });
await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/); await assert.rejects(del("novox", "mesh-catalog", true), /confirm must be the repository's full name/);
await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/); await assert.rejects(del("novox", "mesh-catalog", "novox/mesh-catalog"), /not a repository of the tools' own user/);
@@ -427,6 +444,7 @@ test("gitea_merge_pull_request merges only into a branch whose protection requir
}); });
test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => { test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => {
state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365)
state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true, state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true,
status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true }); status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true });
const set = (args: Record<string, unknown>) => const set = (args: Record<string, unknown>) =>
@@ -0,0 +1,59 @@
// The controller's build-source check asks gitea_branch_protection_get for a repository's rules, which the forge
// answers to a repository's administrator alone. The tools' account is not one (novox/hq ADR 0365 §2), so the read
// goes with the forge's administrator account, and only the read (ADR 0375, issue 772): a protection edit through the
// tools still gets the forge's refusal.
import { test, after } from "node:test";
import assert from "node:assert/strict";
import { createServer } from "node:http";
import { GiteaAdmin, GiteaClient } from "../dist/client.js";
import { ConfiguredToken } from "../dist/token.js";
import { getGiteaTools } from "../dist/tools/index.js";
const adminBasic = "Basic " + Buffer.from("mesh-admin:secret").toString("base64");
const rules = [
{ rule_name: "main", enable_push: false, enable_status_check: true, status_check_contexts: ["mesh/merge-gate"], block_admin_merge_override: true },
{ branch_name: "release", enable_push: false },
];
const seen: string[] = [];
const server = createServer((req, res) => {
const path = (req.url ?? "").replace(/^\/api\/v1/, "");
const as = req.headers.authorization === adminBasic ? "admin" : "tools";
seen.push(`${req.method} ${path} as ${as}`);
const send = (status: number, body: unknown) => { res.writeHead(status, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); };
if (path.includes("/branch_protections")) {
if (as !== "admin") return send(403, { message: "user should be an owner or a collaborator with admin write of a repository" });
if (req.method === "GET") return send(200, rules);
}
send(404, { message: "not found" });
});
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
after(() => new Promise<void>((r) => server.close(() => r())));
const { port } = server.address() as { port: number };
const url = `http://127.0.0.1:${port}`;
const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret"));
const tool = (name: string) => tools.find((t) => t.name === name)!;
test("gitea_branch_protection_get answers every rule, read with the administrator account the tools' token cannot read with", async () => {
const out: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller" });
assert.deepEqual(out.rules.map((r: any) => r.rule), ["main", "release"]);
});
test("gitea_branch_protection_get answers the rule for a branch, null when it has none", async () => {
const main: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "main" });
assert.equal(main.branch, "main");
assert.equal(main.rule.rule, "main");
assert.equal(main.rule.admin_may_override, false);
const none: any = await tool("gitea_branch_protection_get").run({ owner: "novox", repo: "mesh-controller", branch: "dev" });
assert.equal(none.rule, null);
});
test("gitea_branch_protection_set still gets the forge's refusal, as the tools' account", async () => {
await assert.rejects(
tool("gitea_branch_protection_set").run({ owner: "novox", repo: "mesh-controller", branch: "main", status_checks: "mesh/merge-gate" }),
/403/,
);
assert.ok(!seen.some((s) => /^(PATCH|POST|PUT|DELETE) .*branch_protections.* as admin$/.test(s)), seen.join("\n"));
});
+117 -76
View File
@@ -10,12 +10,12 @@
import { test, after } from "node:test"; import { test, after } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { join } from "node:path"; import { join } from "node:path";
import { collectTools } from "@novox/mesh-sdk/tools"; import { collectTools } from "@novox/mesh-sdk/tools";
import { AdminRefused, MINT_RETRY_AFTER_MS, MintedToken, TOKEN_SCOPES } from "../dist/token.js"; import { AdminRefused, MintedToken, TOKEN_SCOPES } from "../dist/token.js";
import { GiteaClient } from "../dist/client.js"; import { GiteaClient } from "../dist/client.js";
import { getGiteaTools } from "../dist/tools/index.js"; import { getGiteaTools } from "../dist/tools/index.js";
@@ -29,9 +29,12 @@ interface Forge {
mints: number; mints: number;
lastScopes: string[] | null; lastScopes: string[] | null;
tokens: Map<string, string>; tokens: Map<string, string>;
/** Tokens beyond the one per name `tokens` holds, by id: what two mints racing leave (novox/hq issue 739). */ /** Tokens beside `tokens` that carry a name already held there: what two mints at once leave behind. */
twins: Map<number, { name: string; sha1: string }>; twins: { id: number; name: string; sha1: string }[];
tokenCalls: number; /** How each DELETE named its token: "id" or "name". */
deletedBy: string[];
/** Plant a token by name, beside any other of the same name, as a raced mint would. */
plant(name: string): void;
scopesOf: Map<string, string[]>; scopesOf: Map<string, string[]>;
admins: Map<string, string>; admins: Map<string, string>;
pullState: string; pullState: string;
@@ -42,12 +45,14 @@ interface Forge {
function fakeForge(): Promise<Forge> { function fakeForge(): Promise<Forge> {
const forge = { const forge = {
mintedFor: "",
mints: 0, mints: 0,
lastScopes: null as string[] | null, lastScopes: null as string[] | null,
tokens: new Map<string, string>(), // name -> value tokens: new Map<string, string>(), // name -> value
twins: new Map<number, { name: string; sha1: string }>(), twins: [] as { id: number; name: string; sha1: string }[],
idOf: new Map<string, number>(), // value -> id deletedBy: [] as string[],
tokenCalls: 0, ids: new Map<string, number>(), // value -> id, as gitea numbers its tokens
nextId: 0,
scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does scopesOf: new Map<string, string[]>(), // value -> scopes, so a route can enforce them like gitea does
admins: new Map([[ADMIN, PASSWORD]]), admins: new Map([[ADMIN, PASSWORD]]),
pullState: "open", pullState: "open",
@@ -79,41 +84,51 @@ function fakeForge(): Promise<Forge> {
const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/); const tokens = url.pathname.match(/^\/api\/v1\/users\/([^/]+)\/tokens(?:\/([^/]+))?$/);
if (tokens) { if (tokens) {
const user = basic(req); const user = basic(req);
forge.tokenCalls++; // gitea's reqSelfOrAdmin: the account itself, or the administrator for another (the tools' account, ADR 0365).
if (user === null || user !== decodeURIComponent(tokens[1])) return json(res, 401, { message: "auth required" }); if (user === null || (user !== decodeURIComponent(tokens[1]) && user !== ADMIN)) return json(res, 401, { message: "auth required" });
const named = (name: string): number => // This fake keeps one account's tokens: the tools' account's when the module mints for it (ADR 0365), and the
(forge.tokens.has(name) ? 1 : 0) + [...forge.twins.values()].filter((t) => t.name === name).length; // admin's own list is then empty — nothing of it is retired.
if (decodeURIComponent(tokens[1]) === ADMIN && forge.mintedFor === "mesh-tools") {
return req.method === "GET" ? json(res, 200, []) : json(res, 404, { message: "token not found" });
}
if (req.method === "POST") forge.mintedFor = decodeURIComponent(tokens[1]);
if (req.method === "POST") { if (req.method === "POST") {
const { name, scopes } = await body(req); const { name, scopes } = await body(req);
if (named(name) > 0) return json(res, 400, { message: "token name has already been used" }); if (forge.tokens.has(name)) return json(res, 400, { message: "token name has already been used" });
forge.mints++; forge.mints++;
forge.lastScopes = scopes; forge.lastScopes = scopes;
const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`; const sha1 = `minted-${forge.mints}-${Math.random().toString(36).slice(2)}`;
forge.tokens.set(name, sha1); forge.tokens.set(name, sha1);
forge.idOf.set(sha1, forge.mints);
forge.scopesOf.set(sha1, scopes); forge.scopesOf.set(sha1, scopes);
return json(res, 201, { id: forge.mints, name, sha1, scopes, token_last_eight: sha1.slice(-8) }); forge.ids.set(sha1, ++forge.nextId);
return json(res, 201, { id: forge.ids.get(sha1), name, sha1, scopes, token_last_eight: sha1.slice(-8) });
} }
// gitea's list: every token of the account, a page at a time (at most three to a page here, so a
// client that reads only the first page misses some), the values never shown.
const listed = (): { id: number; name: string; sha1: string }[] => [
...[...forge.tokens].map(([name, sha1]) => ({ id: forge.ids.get(sha1) ?? 0, name, sha1 })),
...forge.twins,
];
if (req.method === "GET" && !tokens[2]) { if (req.method === "GET" && !tokens[2]) {
const page = Number(url.searchParams.get("page") ?? "1"); const page = Math.max(1, Number(url.searchParams.get("page") ?? "1"));
const all = [...[...forge.tokens].map(([name, v]) => ({ id: forge.idOf.get(v) ?? 0, name })), const limit = Math.min(3, Math.max(1, Number(url.searchParams.get("limit") ?? "3")));
...[...forge.twins].map(([id, t]) => ({ id, name: t.name }))]; const all = listed();
return json(res, 200, page === 1 ? all : []); res.setHeader("X-Total-Count", String(all.length));
return json(res, 200, all.slice((page - 1) * limit, page * limit)
.map((t) => ({ id: t.id, name: t.name, sha1: "", token_last_eight: t.sha1.slice(-8), scopes: [] })));
} }
if (req.method === "DELETE" && tokens[2]) { if (req.method === "DELETE" && tokens[2]) {
// gitea: "identified by ID and if not available by name" — and a name held twice is refused.
const key = decodeURIComponent(tokens[2]); const key = decodeURIComponent(tokens[2]);
// gitea: "identified by ID and if not available by name" const byId = /^\d+$/.test(key);
if (/^\d+$/.test(key)) { const matches = listed().filter((t) => (byId ? String(t.id) === key : t.name === key));
const id = Number(key); if (matches.length === 0) return json(res, 404, { message: "token not found" });
if (forge.twins.delete(id)) return json(res, 204, null); if (matches.length > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
const hit = [...forge.tokens].find(([, v]) => forge.idOf.get(v) === id); const t = matches[0];
if (!hit) return json(res, 404, { message: "token not found" }); forge.deletedBy.push(byId ? "id" : "name");
forge.tokens.delete(hit[0]); const twin = forge.twins.findIndex((w) => w.id === t.id);
return json(res, 204, null); if (twin >= 0) forge.twins.splice(twin, 1);
} else forge.tokens.delete(t.name);
if (named(key) > 1) return json(res, 422, { message: `multiple matches for token name '${key}'` });
if (!forge.tokens.has(key)) return json(res, 404, { message: "token not found" });
forge.tokens.delete(key);
return json(res, 204, null); return json(res, 204, null);
} }
return json(res, 405, { message: "method not allowed" }); return json(res, 405, { message: "method not allowed" });
@@ -179,6 +194,17 @@ function fakeForge(): Promise<Forge> {
{ full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" }, { full_name: "novox/hq", name: "hq", owner: { login: "novox" }, private: true, html_url: "http://fake/novox/hq" },
]); ]);
} }
// What setting up the tools' account asks with the admin login (tools-account.ts, ADR 0365): answered plainly.
if (basic(req) === ADMIN) {
const p = url.pathname;
if (req.method === "GET" && p === "/api/v1/user") return json(res, 200, { login: ADMIN, is_admin: true });
if (req.method === "GET" && p === "/api/v1/users/mesh-tools") return json(res, 200, { login: "mesh-tools", is_admin: false });
if (req.method === "POST" && p === "/api/v1/admin/users") return json(res, 422, { message: "user already exists" });
if (req.method === "PATCH" && p === "/api/v1/admin/users/mesh-tools") return json(res, 200, { login: "mesh-tools" });
if (req.method === "GET" && p === "/api/v1/orgs/novox/teams") return json(res, 200, [{ id: 7, name: "mesh-tools" }]);
if (req.method === "PATCH" && p === "/api/v1/teams/7") return json(res, 200, { id: 7 });
if (req.method === "PUT" && p === "/api/v1/teams/7/members/mesh-tools") return json(res, 204, null);
}
const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/); const adminUser = url.pathname.match(/^\/api\/v1\/admin\/users\/([^/]+)$/);
if (adminUser && req.method === "PATCH") { if (adminUser && req.method === "PATCH") {
const h = req.headers.authorization ?? ""; const h = req.headers.authorization ?? "";
@@ -206,9 +232,15 @@ function fakeForge(): Promise<Forge> {
get pullState() { return forge.pullState; }, get pullState() { return forge.pullState; },
get pullTitle() { return forge.pullTitle; }, get pullTitle() { return forge.pullTitle; },
get branchDeleted() { return forge.branchDeleted; }, get branchDeleted() { return forge.branchDeleted; },
get tokenCalls() { return forge.tokenCalls; },
tokens: forge.tokens, tokens: forge.tokens,
twins: forge.twins, twins: forge.twins,
deletedBy: forge.deletedBy,
plant: (name: string) => {
const sha1 = `planted-${++forge.nextId}`;
forge.ids.set(sha1, forge.nextId);
if (forge.tokens.has(name)) forge.twins.push({ id: forge.nextId, name, sha1 });
else forge.tokens.set(name, sha1);
},
scopesOf: forge.scopesOf, scopesOf: forge.scopesOf,
admins: forge.admins, admins: forge.admins,
close: () => new Promise((r) => server.close(() => r())), close: () => new Promise((r) => server.close(() => r())),
@@ -237,9 +269,8 @@ async function delivered(forge: Forge): Promise<{ env: NodeJS.ProcessEnv; file:
} }
/** A client as a fresh process would build it: a new source over the kept file, its log captured. */ /** A client as a fresh process would build it: a new source over the kept file, its log captured. */
function minted(env: NodeJS.ProcessEnv, logs: string[], now?: () => number): GiteaClient { function minted(env: NodeJS.ProcessEnv, logs: string[]): GiteaClient {
const source = new MintedToken({ const source = new MintedToken({
now,
url: env.MESH_GITEA_URL!, url: env.MESH_GITEA_URL!,
admin: env.MESH_GITEA_ADMIN_USER!, admin: env.MESH_GITEA_ADMIN_USER!,
passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
@@ -354,44 +385,6 @@ test("the kept file is gone but the forge still holds a token by that name: repl
assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n")); assert.ok(logs.some((l) => l.includes('already holds a token named "mesh-tools"')), logs.join("\n"));
}); });
test("two tokens carry the name (two mints raced): each is dropped by its id and one is minted, not a 422 for good (issue 739)", async () => {
const { env, logs } = await delivered(forge);
await minted(env, logs).listRepos();
// What two processes minting at once leave: a second token by the same name, whose value nobody kept.
// Two tokens by the name, neither of them the one kept.
forge.tokens.clear();
forge.twins.set(9001, { name: "mesh-tools", sha1: "raced-first" });
forge.twins.set(9002, { name: "mesh-tools", sha1: "raced-second" });
const repos = await minted(env, logs).listRepos();
assert.equal(repos.length, 1);
assert.equal(forge.twins.size, 0);
assert.equal([...forge.tokens.keys()].filter((n) => n === "mesh-tools").length, 1);
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
});
test("a mint that failed is not asked of the forge again within the retry window: the same error, no basic-auth call (issue 739)", async () => {
const { env, logs } = await delivered(forge);
forge.admins.delete(ADMIN);
try {
let clock = 5_000_000;
const client = minted(env, logs, () => clock);
await assert.rejects(client.listRepos(), AdminRefused);
const calls = forge.tokenCalls;
await assert.rejects(client.listRepos(), AdminRefused);
await assert.rejects(client.listRepos(), AdminRefused);
assert.equal(forge.tokenCalls, calls);
clock += MINT_RETRY_AFTER_MS;
await assert.rejects(client.listRepos(), AdminRefused);
assert.equal(forge.tokenCalls, calls + 1);
} finally {
forge.admins.set(ADMIN, PASSWORD);
}
});
test("concurrent first calls share one mint", async () => { test("concurrent first calls share one mint", async () => {
const { env, logs } = await delivered(forge); const { env, logs } = await delivered(forge);
const client = minted(env, logs); const client = minted(env, logs);
@@ -402,12 +395,61 @@ test("concurrent first calls share one mint", async () => {
assert.equal(forge.mints, before + 1); assert.equal(forge.mints, before + 1);
}); });
test("two tokens hold the name (two mints raced): both are deleted by their id, and one fresh token is minted", async () => {
const { env, file, logs } = await delivered(forge);
forge.tokens.clear();
forge.twins.length = 0;
for (const other of ["laptop", "backup", "ci"]) forge.plant(other); // the account's other tokens: kept
forge.plant("mesh-tools");
forge.plant("mesh-tools"); // gitea refuses to delete either by name from here on
forge.deletedBy.length = 0;
const before = forge.mints;
const repos = await minted(env, logs).listRepos();
assert.equal(repos.length, 1);
assert.equal(forge.mints, before + 1);
assert.deepEqual(forge.deletedBy, ["id", "id"]);
assert.equal(forge.twins.length, 0);
assert.deepEqual([...forge.tokens.keys()].sort(), ["backup", "ci", "laptop", "mesh-tools"]);
assert.equal(await readFile(file, "utf8"), forge.tokens.get("mesh-tools") + "\n");
assert.ok(logs.some((l) => l.includes('holds 2 tokens named "mesh-tools"')), logs.join("\n"));
});
test("two processes on one kept file, both without a token, mint once: the second waits for the first and reuses it", async () => {
const { env, logs } = await delivered(forge);
forge.tokens.clear();
forge.twins.length = 0;
const before = forge.mints;
// Two sources over one state directory, as the tools bundle and the watcher are two processes.
const first = minted(env, logs);
const second = minted(env, logs);
await Promise.all([first.listRepos(), second.listRepos()]);
assert.equal(forge.mints, before + 1);
assert.equal(forge.twins.length, 0);
assert.ok(logs.some((l) => l.includes("another process minted it")), logs.join("\n"));
});
test("a lock left by a process that died mid-mint is taken over, not waited on for ever", async () => {
const { env, file, logs } = await delivered(forge);
forge.tokens.clear();
forge.twins.length = 0;
await mkdir(join(env.MESH_GITEA_STATE_DIR!), { recursive: true });
await writeFile(`${file}.lock`, `2147483646 ${Date.now()} gone\n`); // a pid no process holds
const before = forge.mints;
assert.equal((await minted(env, logs).listRepos()).length, 1);
assert.equal(forge.mints, before + 1);
await assert.rejects(stat(`${file}.lock`), /ENOENT/);
});
test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => { test("the admin account is refused: said plainly, nothing kept, and the next call fails the same way rather than crashing", async () => {
const { env, file, logs } = await delivered(forge); const { env, file, logs } = await delivered(forge);
forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there forge.admins.delete(ADMIN); // the forge's data came from a predecessor; mesh-admin was never created there
try { try {
let clock = 1_000_000; const client = minted(env, logs);
const client = minted(env, logs, () => clock);
const before = forge.mints; const before = forge.mints;
await assert.rejects(client.listRepos(), (err: unknown) => { await assert.rejects(client.listRepos(), (err: unknown) => {
@@ -422,9 +464,8 @@ test("the admin account is refused: said plainly, nothing kept, and the next cal
assert.equal(forge.mints, before); assert.equal(forge.mints, before);
await assert.rejects(stat(file), /ENOENT/); await assert.rejects(stat(file), /ENOENT/);
// The account appears (the operator created it): the next call after the retry window mints and works. // The account appears (the operator created it): the very next call mints and works.
forge.admins.set(ADMIN, PASSWORD); forge.admins.set(ADMIN, PASSWORD);
clock += MINT_RETRY_AFTER_MS;
assert.equal((await client.listRepos()).length, 1); assert.equal((await client.listRepos()).length, 1);
assert.equal(forge.mints, before + 1); assert.equal(forge.mints, before + 1);
} finally { } finally {
+332
View File
@@ -0,0 +1,332 @@
// The forge's tools act as an account of their own, not the site administrator (novox/hq ADR 0365 §2, issue 469):
// the token the tools and the watcher use is minted for `mesh-tools`, a plain account made by the module with the
// administrator's login and given write on the mesh's repositories through one team; the administrator's own
// token by that name goes; a token kept from before, the administrator's, is minted again for the account.
//
// The forge is a fake that keeps accounts, their tokens and the organisation's teams apart, and answers as gitea
// does: an administrator may mint for another account (reqSelfOrAdmin), anyone else only for itself; a protection
// edit needs the repository's administration, which neither a plain account nor its write team holds.
import { test, after } from "node:test";
import assert from "node:assert/strict";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { mkdtemp, readFile, writeFile, mkdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { MintedToken } from "../dist/token.js";
import { GiteaClient } from "../dist/client.js";
import { TOOLS_ACCOUNT, TOOLS_TEAM_UNITS } from "../dist/tools-account.js";
import { repoCheckFacts } from "../dist/pulls.js";
const ADMIN = "mesh-admin";
const PASSWORD = "the-vault-minted-this";
interface Account { admin: boolean; password: string; active: boolean }
interface Token { id: number; name: string; sha1: string; user: string; scopes: string[] }
interface Team { id: number; name: string; units: any; members: Set<string> }
function fakeForge() {
const users = new Map<string, Account>([[ADMIN, { admin: true, password: PASSWORD, active: true }]]);
const tokens: Token[] = [];
const teams: Team[] = [];
const calls: string[] = [];
// What the forge does wrong when a test says so: a token mint or delete it fails, a team another process made
// between this one's look and its create (gitea answers the create 422).
const faults = { mint: false, deleteToken: false, teamRace: false };
let next = 0;
const json = (res: ServerResponse, status: number, body: unknown): void => {
res.writeHead(status, { "Content-Type": "application/json" });
res.end(body === null ? "" : JSON.stringify(body));
};
const read = (req: IncomingMessage): Promise<any> =>
new Promise((resolve) => {
let t = "";
req.on("data", (c) => (t += c));
req.on("end", () => resolve(t ? JSON.parse(t) : null));
});
const who = (req: IncomingMessage): string | null => {
const h = req.headers.authorization ?? "";
if (h.startsWith("Basic ")) {
const [u, p] = Buffer.from(h.slice(6), "base64").toString().split(":");
return users.get(u)?.password === p ? u : null;
}
if (h.startsWith("token ")) return tokens.find((t) => t.sha1 === h.slice(6))?.user ?? null;
return null;
};
const isAdmin = (u: string | null) => u !== null && users.get(u)?.admin === true;
const server = createServer(async (req, res) => {
const url = new URL(req.url ?? "/", "http://fake");
const p = url.pathname.replace(/^\/api\/v1/, "");
const me = who(req);
calls.push(`${req.method} ${p} as ${me}`);
if (me === null) return json(res, 401, { message: "unauthorized" });
let m = p.match(/^\/users\/([^/]+)\/tokens(?:\/(\d+))?$/);
if (m) {
const owner = decodeURIComponent(m[1]);
if (!(req.headers.authorization ?? "").startsWith("Basic ")) return json(res, 401, { message: "basic auth required" });
if (owner !== me && !isAdmin(me)) return json(res, 403, { message: "reqSelfOrAdmin" });
if (!users.has(owner)) return json(res, 404, { message: "user does not exist" });
if (req.method === "POST") {
if (faults.mint) return json(res, 500, { message: "the forge is not well" });
const { name, scopes } = await read(req);
if (tokens.some((t) => t.user === owner && t.name === name)) return json(res, 400, { message: "token name has already been used" });
const t = { id: ++next, name, sha1: `sha-${owner}-${next}`, user: owner, scopes };
tokens.push(t);
return json(res, 201, { id: t.id, name, sha1: t.sha1, scopes });
}
if (req.method === "GET") {
const page = Number(url.searchParams.get("page") ?? "1");
const mine = tokens.filter((t) => t.user === owner);
return json(res, 200, page === 1 ? mine.map((t) => ({ id: t.id, name: t.name, sha1: "" })) : []);
}
if (req.method === "DELETE" && m[2]) {
if (faults.deleteToken && owner === ADMIN) return json(res, 500, { message: "the forge is not well" });
const i = tokens.findIndex((t) => t.user === owner && t.id === Number(m![2]));
if (i < 0) return json(res, 404, { message: "not found" });
tokens.splice(i, 1);
return json(res, 204, null);
}
}
if (req.method === "GET" && p === "/user") return json(res, 200, { login: me, is_admin: isAdmin(me) });
m = p.match(/^\/users\/([^/]+)$/);
if (m && req.method === "GET") {
const u = users.get(decodeURIComponent(m[1]));
if (!u) return json(res, 404, { message: "user does not exist" });
return json(res, 200, { login: m[1], is_admin: u.admin, active: u.active, prohibit_login: false });
}
if (p.startsWith("/admin/") && !isAdmin(me)) return json(res, 403, { message: "must be a site administrator" });
if (req.method === "POST" && p === "/admin/users") {
const b = await read(req);
if (users.has(b.username)) return json(res, 422, { message: "user already exists" });
if (!b.password || b.password.length < 8) return json(res, 422, { message: "password is too short" });
users.set(b.username, { admin: b.admin === true, password: b.password, active: true });
return json(res, 201, { login: b.username, is_admin: b.admin === true });
}
m = p.match(/^\/admin\/users\/([^/]+)$/);
if (m && req.method === "PATCH") {
const u = users.get(decodeURIComponent(m[1]));
if (!u) return json(res, 404, { message: "user does not exist" });
const b = await read(req);
if (b.admin !== undefined) u.admin = b.admin;
if (b.active !== undefined) u.active = b.active;
return json(res, 200, { login: m[1], is_admin: u.admin });
}
if (p === "/orgs/novox/teams") {
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
if (req.method === "GET") return json(res, 200, teams.map((t) => ({ id: t.id, name: t.name })));
const b = await read(req);
const t = { id: ++next, name: b.name, units: b, members: new Set<string>() };
teams.push(t);
if (faults.teamRace) return json(res, 422, { message: "team already exists" });
return json(res, 201, { id: t.id, name: t.name });
}
m = p.match(/^\/teams\/(\d+)(?:\/members\/([^/]+))?$/);
if (m) {
if (!isAdmin(me)) return json(res, 403, { message: "must be an owner of the organisation" });
const t = teams.find((x) => x.id === Number(m![1]));
if (!t) return json(res, 404, { message: "team does not exist" });
if (req.method === "PATCH") { t.units = await read(req); return json(res, 200, { id: t.id }); }
if (req.method === "PUT" && m[2]) { t.members.add(decodeURIComponent(m[2])); return json(res, 204, null); }
}
// Repository administration: a site administrator, or a team member whose team administers the repository.
m = p.match(/^\/repos\/novox\/([^/]+)\/branches\/(.+)$/);
if (m && req.method === "GET") {
if (m[2] === "main") return json(res, 200, { name: "main", protected: true, effective_branch_protection_name: "main",
enable_status_check: true, status_check_contexts: ["mesh/merge-gate", "mesh/repo-check"], user_can_push: false });
if (m[2] === "feat/unguarded") return json(res, 200, { name: m[2], protected: false });
return json(res, 404, { message: "branch does not exist" });
}
// gitea's reqAdmin on the whole group: reading the rules is administration too.
m = p.match(/^\/repos\/novox\/([^/]+)\/branch_protections(?:\/.*)?$/);
if (m) {
const admins = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.permission === "admin");
if (!admins) return json(res, 403, { message: "user should be an owner or a collaborator with admin write of a repository" });
return json(res, 201, { rule_name: "main" });
}
m = p.match(/^\/repos\/novox\/([^/]+)\/statuses\/([0-9a-f]+)$/);
if (m && req.method === "POST") {
const writes = isAdmin(me) || teams.some((t) => t.members.has(me) && t.units.units_map?.["repo.code"] === "write");
if (!writes) return json(res, 403, { message: "write on code required" });
return json(res, 201, { context: (await read(req)).context, creator: { login: me } });
}
if (req.method === "GET" && p === "/repos/search") return json(res, 200, { data: [] });
return json(res, 404, { message: "no such route in the fake" });
});
return new Promise<{ url: string; users: typeof users; tokens: typeof tokens; teams: typeof teams; calls: string[];
faults: typeof faults; close(): Promise<void> }>((resolve) => {
server.listen(0, "127.0.0.1", () => {
const { port } = server.address() as { port: number };
resolve({ url: `http://127.0.0.1:${port}`, users, tokens, teams, calls, faults,
close: () => new Promise((r) => server.close(() => r())) });
});
});
}
const forge = await fakeForge();
after(() => forge.close());
async function environment(): Promise<NodeJS.ProcessEnv & { MESH_GITEA_STATE_DIR: string }> {
const dir = await mkdtemp(join(tmpdir(), "gitea-tools-account-"));
const passwordFile = join(dir, "admin.secret");
await writeFile(passwordFile, PASSWORD + "\n", { mode: 0o600 });
return {
MESH_GITEA_URL: forge.url,
MESH_GITEA_ADMIN_USER: ADMIN,
MESH_GITEA_ADMIN_PASSWORD_FILE: passwordFile,
MESH_GITEA_STATE_DIR: join(dir, "state"),
};
}
/** A source as a fresh process builds it from the manifest's environment (fromEnv shares one per file). */
function fresh(env: NodeJS.ProcessEnv): GiteaClient {
return GiteaClient.fromEnv(env);
}
test("the tools' token is minted for an account of their own, which is no administrator and writes through its team", async () => {
const env = await environment();
await fresh(env).listRepos();
const account = forge.users.get(TOOLS_ACCOUNT);
assert.ok(account, "the module made the tools' account");
assert.equal(account!.admin, false, "the tools' account is no administrator");
const held = forge.tokens.filter((t) => t.name === "mesh-tools");
assert.deepEqual(held.map((t) => t.user), [TOOLS_ACCOUNT], "the one token is the tools' account's, none is the administrator's");
assert.equal(await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8"), held[0].sha1 + "\n");
const team = forge.teams.find((t) => t.name === "mesh-tools");
assert.ok(team, "the team is made in the mesh's organisation");
assert.ok(team!.members.has(TOOLS_ACCOUNT));
assert.equal(team!.units.permission, "write");
assert.equal(team!.units.includes_all_repositories, true);
assert.equal(team!.units.can_create_org_repo, false, "a repository's maker would administer it");
assert.deepEqual(team!.units.units_map, TOOLS_TEAM_UNITS.units_map);
assert.ok(!Object.values(team!.units.units_map).includes("admin"));
});
test("with the tools' token, the forge writes a status and refuses a protection edit — the forge itself, not a guard", async () => {
const env = await environment();
const client = fresh(env);
await client.setCommitStatus("novox", "hq", "abc123def4567890abc123def4567890abc12345", { state: "success", context: "mesh/merge-gate", description: "ok" } as any);
await assert.rejects(
client.setBranchProtection("novox", "hq", "main", { statusChecks: ["mesh/merge-gate"] }),
/403/,
);
});
test("what a base's protection asks is read from the branch, which the tools' account may read, not from the rules' list it may not", async () => {
const env = await environment();
const client = fresh(env);
await assert.rejects(client.branchProtections("novox", "hq"), /403/);
const rule = await client.appliedProtection("novox", "hq", "main");
assert.deepEqual(rule?.status_check_contexts, ["mesh/merge-gate", "mesh/repo-check"]);
assert.equal(rule?.enable_status_check, true);
assert.equal(rule?.rule_name, "main");
assert.equal(await client.appliedProtection("novox", "hq", "feat/unguarded"), null);
assert.equal(await client.appliedProtection("novox", "hq", "no/such-branch"), null);
});
test("the administrator's token from before goes: a kept admin token is minted again for the tools' account, and the forge drops the old one", async () => {
const env = await environment();
// What the module kept before ADR 0365: a token of the administrator's own, by the same name, its scopes beside it.
const old = { id: 900, name: "mesh-tools", sha1: "sha-admin-old", user: ADMIN, scopes: ["write:repository", "write:issue", "read:user"] };
forge.tokens.push(old);
await mkdir(env.MESH_GITEA_STATE_DIR, { recursive: true });
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token"), old.sha1 + "\n", { mode: 0o600 });
await writeFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "read:user,write:issue,write:repository\n", { mode: 0o600 });
await fresh(env).listRepos();
assert.ok(!forge.tokens.some((t) => t.user === ADMIN), "the administrator holds no tools' token any more");
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
assert.notEqual(kept, old.sha1);
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT);
assert.match(await readFile(join(env.MESH_GITEA_STATE_DIR, "token.scopes"), "utf8"), / for mesh-tools\n$/);
});
test("a tools' account someone made an administrator is made a plain account again at the next mint", async () => {
const env = await environment();
forge.users.get(TOOLS_ACCOUNT)!.admin = true;
await fresh(env).listRepos();
assert.equal(forge.users.get(TOOLS_ACCOUNT)!.admin, false);
});
test("the tools never act as the administrator: MESH_GITEA_TOOLS_USER naming it is refused", async () => {
const env = await environment();
assert.throws(() => MintedToken.fromEnv(forge.url, { ...env, MESH_GITEA_TOOLS_USER: ADMIN }), /names the admin account/);
});
test("the manifest hands the tools their account, and the merge-ready authors name it rather than the administrator", async () => {
const manifest = JSON.parse(await readFile(new URL("../module.json", import.meta.url), "utf8"));
const code = manifest.build.artifacts.find((a: any) => a.name === "code");
assert.equal(code.env.MESH_GITEA_TOOLS_USER, TOOLS_ACCOUNT);
const authors = String(manifest.settings["ready-authors"].default).split(/\s+/);
assert.ok(authors.includes(TOOLS_ACCOUNT));
// Kept for now (review of issue 469): a pull request the admin account marked ready before the tools acted as
// their own keeps counting as ready after the delivery.
assert.ok(authors.includes(ADMIN), "marks made by the admin account before ADR 0365 keep counting");
// The Go forge bundle reads the token the tools' account holds, never the administrator's password.
const forgeBundle = manifest.build.artifacts.find((a: any) => a.name === "forge");
assert.ok(forgeBundle.env.MESH_GITEA_TOKEN_FILE);
assert.equal(forgeBundle.env.MESH_GITEA_ADMIN_PASSWORD_FILE, undefined);
});
// Review of issue 469: what reads the rules' list is moved to the branch, and nothing leaves the tools without a token.
test("a repository check said as a warning on a head without merge-check.sh reads the base's protection from the branch", async () => {
const client = fresh(await environment());
const forgeOf = { holdsFile: async () => false, appliedProtection: client.appliedProtection.bind(client) };
const c = { owner: "novox", repo: "hq", commit: "abc123def4567890abc123def4567890abc12345", "repo-check": { verdict: "warning" } } as any;
// main requires mesh/repo-check: required, so a head that defines no check fails it for a person to see.
assert.deepEqual(await repoCheckFacts(forgeOf, c, "main"), { defined: false, required: true });
// A base nothing guards: not required, so the head's repository check passes rather than fails.
assert.deepEqual(await repoCheckFacts(forgeOf, c, "feat/unguarded"), { defined: false, required: false });
});
test("a mint that fails leaves the administrator's old token in place: the tools are never left without one", async () => {
const env = await environment();
const old = { id: 901, name: "mesh-tools", sha1: "sha-admin-kept", user: ADMIN, scopes: ["read:user"] };
forge.tokens.push(old);
forge.faults.mint = true;
try {
await assert.rejects(fresh(env).listRepos(), /500/);
} finally {
forge.faults.mint = false;
}
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's token was deleted before a new one was kept");
forge.tokens.splice(forge.tokens.indexOf(old), 1);
});
test("an old token that cannot be deleted after the mint is said, the new token kept, and deleted at the next start", async () => {
const env = await environment();
const old = { id: 902, name: "mesh-tools", sha1: "sha-admin-stuck", user: ADMIN, scopes: ["read:user"] };
forge.tokens.push(old);
forge.faults.deleteToken = true;
try {
await fresh(env).listRepos();
} finally {
forge.faults.deleteToken = false;
}
const kept = (await readFile(join(env.MESH_GITEA_STATE_DIR, "token"), "utf8")).trim();
assert.equal(forge.tokens.find((t) => t.sha1 === kept)?.user, TOOLS_ACCOUNT, "the account's token is kept and works");
assert.ok(forge.tokens.some((t) => t.sha1 === old.sha1));
// The next process reuses the kept token and retires the old one it could not before.
const next = new MintedToken({ url: forge.url, admin: ADMIN, passwordFile: env.MESH_GITEA_ADMIN_PASSWORD_FILE!,
file: join(env.MESH_GITEA_STATE_DIR, "token"), account: TOOLS_ACCOUNT, log: () => {} });
assert.equal(await next.current(), kept);
assert.ok(!forge.tokens.some((t) => t.sha1 === old.sha1), "the admin's old token was deleted at the next start");
});
test("a team another process made between the look and the create is found, not a failure", async () => {
forge.teams.splice(0);
forge.faults.teamRace = true;
try {
await fresh(await environment()).listRepos();
} finally {
forge.faults.teamRace = false;
}
const team = forge.teams.find((t) => t.name === "mesh-tools");
assert.ok(team?.members.has(TOOLS_ACCOUNT));
});
+225 -10
View File
@@ -83,20 +83,235 @@ test("each forge event is named in the module's stream and trimmed", async () =>
assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other"); assert.equal(eventOf("pull_request_review_approved", {}).name, "forge.other");
}); });
test("the system webhook is made when none points at the receiver, and corrected when one does", async () => { // A forge's admin hooks as 1.27.3 keeps them: POST /admin/hooks makes a system webhook only when config.is_system_webhook
const { ensureSystemHook } = await import("../webhook.ts"); // is "true", and a default webhook otherwise; the listing names one kind at a time (type=system, the default, or default).
// Repositories hold their own webhooks (a default webhook's copies), listed a page at a time.
function fakeForge(start: { id: number; url: string; system: boolean }[] = [],
repoHooks: Record<string, { id: number; url: string }[]> = {}, repoCount = 0) {
const hooks = start.map((h) => ({ ...h, events: ["push"], active: true }));
const repos = [...Object.keys(repoHooks)];
for (let i = 0; repos.length < repoCount; i++) repos.push(`novox/r${i}`);
let next = 100;
const calls: string[] = []; const calls: string[] = [];
let hooks: unknown[] = []; const made: any[] = [];
const page = (items: any[], q: URLSearchParams) => {
const p = Number(q.get("page") ?? 1), l = Number(q.get("limit") ?? 50);
return items.slice((p - 1) * l, p * l);
};
const admin = { call: async (method: string, path: string, body?: any) => { const admin = { call: async (method: string, path: string, body?: any) => {
calls.push(`${method} ${path}`); calls.push(`${method} ${path}`);
if (method === "GET") return { status: 200, body: hooks }; const u = new URL(path, "http://f");
if (method === "POST") { assert.ok(body.events.includes("pull_request") && body.events.includes("push")); assert.equal(body.config.secret, secret); return { status: 201, body: {} }; } const repoHook = u.pathname.match(/^\/repos\/([^/]+\/[^/]+)\/hooks(?:\/(\d+))?$/);
return { status: 200, body: {} }; if (method === "GET" && u.pathname === "/repos/search") {
return { status: 200, body: { ok: true, data: page(repos.map((full_name) => ({ full_name })), u.searchParams) } };
}
if (repoHook) {
const list = repoHooks[repoHook[1]] ?? [];
if (method === "GET") return { status: 200, body: page(list.map((h) => ({ id: h.id, config: { url: h.url } })), u.searchParams) };
const i = list.findIndex((h) => h.id === Number(repoHook[2]));
if (i < 0) return { status: 404, body: {} };
list.splice(i, 1);
return { status: 204, body: null };
}
if (method === "GET") {
const type = u.searchParams.get("type") ?? "system";
const listed = hooks.filter((h) => type === "all" || h.system === (type === "system"));
return { status: 200, body: listed.map((h) => ({ id: h.id, active: h.active, events: h.events,
config: { url: h.url, content_type: "json" } })) };
}
if (method === "POST") {
made.push(body);
hooks.push({ id: next++, url: body.config.url, system: body.config.is_system_webhook === "true", events: body.events,
active: body.active });
return { status: 201, body: {} };
}
if (method === "DELETE") {
const id = Number(path.split("/").pop());
const i = hooks.findIndex((h) => h.id === id);
if (i < 0) return { status: 404, body: {} };
hooks.splice(i, 1);
return { status: 204, body: null };
}
return { status: 405, body: {} };
} }; } };
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "created"); return { admin, hooks, calls, made, repoHooks };
hooks = [{ id: 4, config: { url: "http://h:3047/forge-event" } }]; }
assert.equal(await ensureSystemHook(admin, "http://h:3047/forge-event", secret), "corrected");
assert.ok(calls.includes("PATCH /admin/hooks/4")); test("the webhook is made a system webhook, which the forge sends every repository's events to", async () => {
const { ensureSystemHook } = await import("../webhook.ts");
const forge = fakeForge();
const url = "http://h:3047/forge-event";
const hook = await ensureSystemHook(forge.admin, url, secret);
assert.equal(forge.made.length, 1);
assert.equal(forge.made[0].config.is_system_webhook, "true", "without it the forge makes a default webhook, which fires for no repository");
assert.equal(forge.made[0].config.secret, secret);
assert.ok(forge.made[0].events.includes("pull_request") && forge.made[0].events.includes("push"));
assert.deepEqual(forge.hooks.map((h) => h.system), [true]);
assert.equal(hook.kind, "system");
assert.equal(hook.did, "created");
assert.equal(hook.url, url);
assert.ok(hook.events.includes("pull_request"));
});
test("default webhooks left at the receiver by an earlier start are replaced by one system webhook", async () => {
const { ensureSystemHook } = await import("../webhook.ts");
const url = "http://h:3047/forge-event";
// What the start at 23:05 left: a default webhook, invisible to the system listing, made again by every start.
const forge = fakeForge([{ id: 4, url, system: false }, { id: 5, url, system: false }, { id: 6, url: "http://other/x", system: true }]);
const hook = await ensureSystemHook(forge.admin, url, secret);
assert.deepEqual(hook.removed, { system: 0, default: 2, repository: 0, repositories: [] });
assert.equal(hook.did, "replaced");
assert.deepEqual(forge.hooks.map((h) => [h.url, h.system]), [["http://other/x", true], [url, true]], "another url is kept");
// A second start replaces the system webhook, so the secret kept now is the one the forge signs with (its edit never
// changes a secret).
const again = await ensureSystemHook(forge.admin, url, secret);
assert.deepEqual(again.removed, { system: 1, default: 0, repository: 0, repositories: [] });
assert.equal(forge.hooks.filter((h) => h.url === url).length, 1);
assert.ok(forge.calls.includes("DELETE /admin/hooks/4") && forge.calls.includes("DELETE /admin/hooks/5"));
// The new one is made before any old one is deleted, so no event falls in a gap.
const post = forge.calls.indexOf("POST /admin/hooks");
assert.ok(post >= 0 && forge.calls.findIndex((c) => c.startsWith("DELETE")) > post);
});
test("a repository's copy of a default webhook at the receiver is deleted, on every page, and said", async () => {
const { ensureSystemHook } = await import("../webhook.ts");
const url = "http://h:3047/forge-event";
// A repository made after the default webhook got its copy, which would send its events a second time; 120
// repositories, so the copies sit beyond the first page.
const repoHooks: Record<string, { id: number; url: string }[]> = {};
for (let i = 0; i < 118; i++) repoHooks[`novox/r${i}`] = [];
repoHooks["novox/late"] = [{ id: 31, url }, { id: 32, url: "http://ci/hook" }];
repoHooks["novox/later"] = [{ id: 33, url }];
const forge = fakeForge([{ id: 4, url, system: false }], repoHooks);
const hook = await ensureSystemHook(forge.admin, url, secret);
assert.deepEqual(hook.removed, { system: 0, default: 1, repository: 2, repositories: ["novox/late", "novox/later"] });
assert.deepEqual(forge.repoHooks["novox/late"].map((h) => h.id), [32], "a repository's other webhook is kept");
assert.deepEqual(forge.repoHooks["novox/later"], []);
assert.ok(forge.calls.some((c) => /\/repos\/search\?.*page=3/.test(c)), "every page of repositories is read");
});
test("a forge that makes no system webhook is said, never taken as heard", async () => {
const { ensureSystemHook } = await import("../webhook.ts");
const forge = fakeForge();
const admin = { call: async (method: string, path: string, body?: any) => {
// A forge that ignores the flag: what 23:05 met, had the module read back.
if (method === "POST") body = { ...body, config: { ...body.config, is_system_webhook: undefined } };
return forge.admin.call(method, path, body);
} };
await assert.rejects(ensureSystemHook(admin, "http://h:3047/forge-event", secret), /names 0 new system webhooks/);
});
test("the receiver tells what it verified and what it refused", async () => {
const { receiver, HOOK_PATH } = await import("../webhook.ts");
const heard: string[] = [], refused: string[] = [];
const server = receiver({ secret, port: 0, log: () => {}, publish: async () => {},
heard: (kind, delivery) => heard.push(`${kind} ${delivery}`), refused: (reason) => refused.push(reason) });
await new Promise<void>((r) => server.listen(0, "127.0.0.1", () => r()));
const url = `http://127.0.0.1:${(server.address() as AddressInfo).port}${HOOK_PATH}`;
try {
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened), "X-Gitea-Delivery": "d-9" } });
await fetch(url, { method: "POST", body: opened, headers: { "X-Gitea-Event": "pull_request", "X-Gitea-Signature": sign(opened, "x") } });
await server.idle();
assert.deepEqual(heard, ["pull_request d-9"]);
assert.deepEqual(refused, ["its signature does not verify"]);
} finally { server.close(); }
});
test("a forge that never delivers is said loudly once, and unhealthy, when the poll finds what no event said", async () => {
const { Pace } = await import("../webhook.ts");
let now = 1_000_000;
const said: string[] = [];
const pace = new Pace(() => now, (l) => said.push(l));
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
pace.listening("172.17.0.1", 3047, hook.url, hook);
now += 5 * 60_000;
pace.miss("novox/hq#7 at aaaaaaaa");
assert.equal(said.length, 0, "five minutes after the start is not yet silence");
assert.equal((pace.report() as any).healthy, true);
pace.refused("it is unsigned", "172.17.0.3");
assert.equal((pace.report() as any).healthy, false, "refused, and none verified");
now += 6 * 60_000;
pace.miss("the merge of novox/hq#7");
pace.miss("novox/hq#8 at bbbbbbbb");
assert.equal(said.filter((l) => /NOT HEARD: no delivery ever verified/.test(l)).length, 1, "said once");
let r = pace.report() as any;
assert.equal(r.healthy, false);
assert.equal(r.state, "unhealthy");
assert.match(r.why, /never delivered/);
assert.equal(r.polls.found_unsaid, 3);
assert.equal(r.refused.count, 1);
assert.equal(r.refused.last.reason, "it is unsigned");
assert.equal(r.hook.kind, "system");
assert.equal(r.receiver.port, 3047);
// The first delivery clears it.
now += 1000;
pace.heard("push", "d-1");
r = pace.report() as any;
assert.equal(r.healthy, true);
assert.equal(r.verified.count, 1);
assert.equal(r.verified.last.event, "push");
assert.equal(r.polls.every_seconds, 300);
assert.match(said.join("\n"), /heard again/);
});
test("a forge whose events stop after one delivery is said loudly once per silent spell, and refusals are unhealthy", async () => {
const { Pace } = await import("../webhook.ts");
let now = 1_000_000;
const said: string[] = [];
const pace = new Pace(() => now, (l) => said.push(l));
const hook = { kind: "system" as const, id: 7, url: "http://h:3047/forge-event", events: ["push"], active: true,
did: "created" as const, removed: { system: 0, default: 0, repository: 0, repositories: [] } };
pace.listening("172.17.0.1", 3047, hook.url, hook);
now += 60_000;
pace.heard("push", "d-1");
// The hook is deleted by hand: nothing more comes, and the poll finds what no event said.
now += 11 * 60_000;
pace.miss("novox/hq#7 at aaaaaaaa");
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 0, "one find alone may be the poll beating an event");
pace.miss("the merge of novox/hq#7");
pace.miss("novox/hq#8 at bbbbbbbb");
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 1, "said once");
assert.match(said.find((l) => /NOT HEARD/.test(l)) ?? "", /since the last at/);
assert.equal((pace.report() as any).healthy, false);
// A delivery ends the spell; the next spell is said again.
now += 1000;
pace.heard("push", "d-2");
assert.equal((pace.report() as any).healthy, true);
now += 11 * 60_000;
pace.miss("novox/hq#9 at cccccccc");
pace.miss("novox/hq#10 at dddddddd");
assert.equal(said.filter((l) => /NOT HEARD/.test(l)).length, 2, "a second silent spell is said again");
// Every delivery refused after a verified one (a secret that no longer matches) is unhealthy at once.
now += 1000;
pace.heard("push", "d-3");
now += 1000;
pace.refused("its signature does not verify", "172.17.0.3");
pace.refused("its signature does not verify", "172.17.0.3");
const r = pace.report() as any;
assert.equal(r.healthy, false);
assert.equal(r.refused.since_verified, 2);
assert.match(r.why, /2 deliveries refused since the last verified one.*does not verify/);
now += 1000;
pace.heard("push", "d-4");
assert.equal((pace.report() as any).healthy, true, "a verified delivery clears it");
});
test("without a receiver by configuration the polls alone watch, healthy; a receiver that failed is unhealthy", async () => {
const { Pace } = await import("../webhook.ts");
let now = 1_000_000;
const said: string[] = [];
const quiet = new Pace(() => now, (l) => said.push(l));
quiet.unheard("MESH_GITEA_HOOK_URL unset", false);
now += 20 * 60_000;
quiet.miss("novox/hq#7 at aaaaaaaa");
assert.equal(said.length, 0, "no deliveries were expected");
assert.equal((quiet.report() as any).healthy, true);
const failed = new Pace(() => now, () => {});
failed.unheard("the forge's system webhook could not be made: 403");
const r = failed.report() as any;
assert.equal(r.healthy, false);
assert.match(r.why, /403/);
}); });
test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => { test("the polls look every 30 s until events are heard, every five minutes while they are, and 30 s again on a miss", async () => {
+208 -78
View File
@@ -12,7 +12,20 @@
// hands a token's value out exactly once, so a token not kept is a token lost; // hands a token's value out exactly once, so a token not kept is a token lost;
// - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not // - re-minted when the forge rejects it (401) or the kept file is gone. The one case that is not
// a fault: the forge's data was restored from a predecessor and the token the file names never // a fault: the forge's data was restored from a predecessor and the token the file names never
// existed there. // existed there;
// - minted by one process at a time. The runtime starts each of the bundle's entrypoints as a process
// of its own (novox/hq ADR 0193): the watcher (index.js) and the tools (tools/index.js) are two
// processes over one kept file. Two that minted at once left the forge two tokens by one name, which
// gitea then refuses to delete by name — for ever, and every forge tool with it (the 2026-10-14
// outage). So a mint holds a lock file beside the kept token, and reads the kept file again once it
// holds it: what another process minted meanwhile is reused, not minted over;
// - replaced by id: an old token by this name is found in the account's list and deleted by its id,
// every one of them, so a name held twice is cleared rather than refused.
//
// - minted for the tools' account, not the administrator (novox/hq ADR 0365 §2, issue 469): the administrator's
// login mints it for `mesh-tools` (tools-account.ts), after making that account and its team, and deletes any
// token by this name the administrator itself still holds — the token the tools used before. A token kept
// for another account than the one asked (the scopes line says whose) is minted again at the next start.
// //
// An explicitly configured token still wins, and is never minted over: if it is rejected, that is // An explicitly configured token still wins, and is never minted over: if it is rejected, that is
// reported, not repaired — somebody chose it. // reported, not repaired — somebody chose it.
@@ -20,8 +33,10 @@
// The token is never logged. Lines say that one was minted, reused or renewed, and where it is // The token is never logged. Lines say that one was minted, reused or renewed, and where it is
// kept; never what it is. // kept; never what it is.
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
import { randomBytes } from "node:crypto";
import { dirname, join } from "node:path"; import { dirname, join } from "node:path";
import { setUpToolsAccount, TOOLS_ACCOUNT, type AdminCall } from "./tools-account.js";
/** The name the token carries in the forge's own list — one per mesh runtime, found by name. */ /** The name the token carries in the forge's own list — one per mesh runtime, found by name. */
export const TOKEN_NAME = "mesh-tools"; export const TOKEN_NAME = "mesh-tools";
@@ -39,7 +54,7 @@ export const TOKEN_NAME = "mesh-tools";
* `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the * `gitea_api` (novox/hq 229); whoever may call a tool includes agents, and the forge's administration is the
* operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token. * operator's. The provisioner settles users with the admin account's own login (GiteaAdmin), never this token.
* A token kept with a different list is minted again with this one at the next start (the scopes are kept * A token kept with a different list is minted again with this one at the next start (the scopes are kept
* beside it), and the forge drops the old one by name — so the scope a kept token had does not outlive this. * beside it), and the old one is deleted from the forge by its id — so the scope a kept token had does not outlive this.
* *
* A token kept from before a scope was added lacks it: the forge answers such a call with * A token kept from before a scope was added lacks it: the forge answers such a call with
* `403 token does not have at least one of required scope(s)`, and the client treats that like a * `403 token does not have at least one of required scope(s)`, and the client treats that like a
@@ -95,42 +110,37 @@ export interface MintedTokenOptions {
readonly file: string; readonly file: string;
readonly name?: string; readonly name?: string;
readonly scopes?: readonly string[]; readonly scopes?: readonly string[];
/**
* The account the token is minted for: the tools' own (novox/hq ADR 0365 §2). The admin login mints it, after
* `setUp` made the account, and the admin's own tokens by this name are deleted. Left out, the token is the
* admin's own, as it was before.
*/
readonly account?: string;
/** Run with the admin's password before each mint for `account`: makes the account and its team (tools-account.ts). */
readonly setUp?: (password: string) => Promise<void>;
readonly log?: (line: string) => void; readonly log?: (line: string) => void;
readonly fetch?: typeof fetch; readonly fetch?: typeof fetch;
/** The clock, in milliseconds; a test sets it. */
readonly now?: () => number;
} }
/**
* **How long a failed mint is not tried again** (novox/hq issue 739). Every call that finds no token mints, and a
* mint is two or three calls over basic auth, each one hashing the admin password on the forge. A mint that keeps
* failing, once per tool call, is a loop the forge pays for in CPU: on 2026-10-14 it ran several times a minute
* on the control-node while it was already overloaded. Within this window the last failure is answered again,
* without asking the forge.
*/
export const MINT_RETRY_AFTER_MS = 30_000;
/** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */ /** The token the module mints for itself, kept in its state and renewed when the forge rejects it. */
export class MintedToken implements TokenSource { export class MintedToken implements TokenSource {
private held: string | null = null; private held: string | null = null;
private readFile = false; private readFile = false;
private inflight: Promise<string> | null = null; private inflight: Promise<string> | null = null;
private failed: { at: number; error: unknown } | null = null;
private readonly name: string; private readonly name: string;
private readonly scopes: readonly string[]; private readonly scopes: readonly string[];
private readonly account: string;
private readonly log: (line: string) => void; private readonly log: (line: string) => void;
private readonly fetchImpl: typeof fetch; private readonly fetchImpl: typeof fetch;
constructor(private readonly opts: MintedTokenOptions) { constructor(private readonly opts: MintedTokenOptions) {
this.name = opts.name ?? TOKEN_NAME; this.name = opts.name ?? TOKEN_NAME;
this.scopes = opts.scopes ?? TOKEN_SCOPES; this.scopes = opts.scopes ?? TOKEN_SCOPES;
this.account = opts.account ?? opts.admin;
this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`)); this.log = opts.log ?? ((line) => console.log(`[gitea] ${line}`));
this.fetchImpl = opts.fetch ?? fetch; this.fetchImpl = opts.fetch ?? fetch;
this.now = opts.now ?? Date.now;
} }
private readonly now: () => number;
/** /**
* Build from the runtime's environment: the forge's URL, the admin login and password file the * Build from the runtime's environment: the forge's URL, the admin login and password file the
* manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory * manifest hands the runtime, and the module's state directory (MESH_GITEA_STATE_DIR, a directory
@@ -139,12 +149,9 @@ export class MintedToken implements TokenSource {
*/ */
static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken { static fromEnv(url: string, env: NodeJS.ProcessEnv = process.env): MintedToken {
const opts = MintedToken.optionsFromEnv(url, env); const opts = MintedToken.optionsFromEnv(url, env);
// One source per kept file in a process. The watcher and the tools entrypoint both build a // One source per kept file in a process: two sources in one process would each renew on a 401.
// client in the same runtime; two sources over one file would each renew on a 401 and drop the // This covers one process only. The watcher and the tools are two processes (the runtime starts
// other's token by name, forever. Shared, a renewal is one renewal. Across processes it is not shared: the // each entrypoint as its own), and those are kept apart by the mint's lock file (mint, below).
// runtime serves the watcher (index.js) and the tools (tools/index.js) as separate bundles, so two sources over
// one file do exist. renew() reads the kept file before it mints, and a race that still leaves two tokens by
// the name is undone by dropping each by its id (doMint; novox/hq issue 739).
const shared = MintedToken.shared.get(opts.file); const shared = MintedToken.shared.get(opts.file);
if (shared) return shared; if (shared) return shared;
const source = new MintedToken(opts); const source = new MintedToken(opts);
@@ -166,7 +173,17 @@ export class MintedToken implements TokenSource {
if (missing.length) { if (missing.length) {
throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`); throw new Error(`no Gitea token, and nothing to mint one with — set MESH_GITEA_TOKEN, or ${missing.join(", ")}`);
} }
return { url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token") }; // The tools' own account, never the administrator (novox/hq ADR 0365 §2): MESH_GITEA_TOOLS_USER names it.
const account = env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT;
if (account === admin) throw new Error(`MESH_GITEA_TOOLS_USER names the admin account "${admin}"; the tools act as an account of their own (novox/hq ADR 0365)`);
return {
url, admin: admin!, passwordFile: passwordFile!, file: join(stateDir!, "token"), account,
// GiteaAdmin is loaded when first used: client.ts imports this file.
setUp: async (password) => {
const { GiteaAdmin } = await import("./client.js");
await setUpToolsAccount(new GiteaAdmin(url, admin!, password), account, () => randomBytes(24).toString("base64url"));
},
};
} }
async current(): Promise<string> { async current(): Promise<string> {
@@ -177,11 +194,13 @@ export class MintedToken implements TokenSource {
if (kept !== null && this.keptScopes() === this.scopesLine()) { if (kept !== null && this.keptScopes() === this.scopesLine()) {
this.held = kept; this.held = kept;
this.log(`reusing the token kept at ${this.opts.file}`); this.log(`reusing the token kept at ${this.opts.file}`);
// A retirement of the admin's old token that failed after the mint is tried again, once a process.
if (existsSync(this.retireFile)) await this.locked(() => this.retireAdmin()).catch((err) => this.log(`${err}`));
return kept; return kept;
} }
if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them"); if (kept !== null) return this.mint("the kept token was minted with other scopes than the tools take now — minting one with them", kept);
} }
return this.mint(this.failed ? "no token held since the last mint failed — minting one" : "no token kept — minting one"); return this.mint("no token kept — minting one", null);
} }
async renew(rejected: string): Promise<string> { async renew(rejected: string): Promise<string> {
@@ -195,7 +214,7 @@ export class MintedToken implements TokenSource {
return kept; return kept;
} }
this.held = null; this.held = null;
return this.mint("the forge rejected the kept token — minting a fresh one"); return this.mint("the forge rejected the kept token — minting a fresh one", rejected);
} }
/** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */ /** What the forge's scoped tokens say when a kept token predates a scope the tools now need. */
@@ -203,28 +222,95 @@ export class MintedToken implements TokenSource {
return status === 403 && /required scope/i.test(body); return status === 403 && /required scope/i.test(body);
} }
/** One mint at a time: concurrent first calls share it, rather than each minting its own. */ /**
private mint(why: string): Promise<string> { * One mint at a time: concurrent first calls in this process share it, and a mint in another process
* on this machine is waited for (the lock file). Once the lock is held, the kept file is read again:
* a token there that is not `stale` (the one rejected, or kept with other scopes) and carries the
* scopes asked for was minted by the process that held the lock before — it is reused.
*/
private mint(why: string, stale: string | null): Promise<string> {
if (this.inflight === null) { if (this.inflight === null) {
// A mint that failed a moment ago is not asked of the forge again yet (novox/hq issue 739): its error stands.
if (this.failed !== null && this.now() - this.failed.at < MINT_RETRY_AFTER_MS) return Promise.reject(this.failed.error);
this.log(why); this.log(why);
this.inflight = this.doMint().then( this.inflight = this.locked(async () => {
(token) => { const kept = this.read();
this.failed = null; if (kept !== null && kept !== stale && this.keptScopes() === this.scopesLine()) {
return token; this.held = kept;
}, this.log(`another process minted it meanwhile — reusing the token kept at ${this.opts.file}`);
(err) => { return kept;
this.failed = { at: this.now(), error: err }; }
throw err; return this.doMint();
}, }).finally(() => {
).finally(() => {
this.inflight = null; this.inflight = null;
}); });
} }
return this.inflight; return this.inflight;
} }
/** How long a mint may hold the lock before another process takes it as left behind, and how long one waits. */
static readonly LOCK_STALE_MS = 120_000;
static readonly LOCK_WAIT_MS = 150_000;
/**
* Run `work` holding the lock file beside the kept token: created exclusively, holding this process's
* pid, the time and a nonce; removed after, only when it is still ours. A lock whose process is gone,
* or older than LOCK_STALE_MS, was left by a process that died mid-mint and is taken over — never
* waited on for ever, which would be the same lock-out by another road.
*/
private async locked<T>(work: () => Promise<T>): Promise<T> {
const lock = `${this.opts.file}.lock`;
mkdirSync(dirname(this.opts.file), { recursive: true, mode: 0o700 });
const mine = `${process.pid} ${Date.now()} ${randomBytes(8).toString("hex")}\n`;
const deadline = Date.now() + MintedToken.LOCK_WAIT_MS;
for (;;) {
try {
const fd = openSync(lock, "wx", 0o600);
try { writeSync(fd, mine); } finally { closeSync(fd); }
break;
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "EEXIST") {
throw new Error(`cannot take the mint lock at ${lock}: ${(err as Error).message}`);
}
}
if (this.leftBehind(lock)) {
this.log(`the mint lock at ${lock} was left by a process that is gone — taking it over`);
try { unlinkSync(lock); } catch { /* another waiter took it over first */ }
continue;
}
if (Date.now() > deadline) {
throw new Error(`another process has held the mint lock at ${lock} for over ${MintedToken.LOCK_WAIT_MS / 1000}s — not minting beside it`);
}
await new Promise((r) => setTimeout(r, 50 + Math.floor(Math.random() * 50)));
}
try {
return await work();
} finally {
try {
if (readFileSync(lock, "utf8") === mine) unlinkSync(lock);
} catch { /* already gone */ }
}
}
/** Whether the lock's holder is gone: its pid answers no signal, or it is older than LOCK_STALE_MS. */
private leftBehind(lock: string): boolean {
let text: string;
let age: number;
try {
text = readFileSync(lock, "utf8");
age = Date.now() - statSync(lock).mtimeMs;
} catch {
return false; // released meanwhile: the next attempt takes it
}
if (age > MintedToken.LOCK_STALE_MS) return true;
const pid = Number(text.split(" ")[0]);
if (!Number.isInteger(pid) || pid <= 0) return age > 1_000; // half-written, or not ours: give it a moment
try {
process.kill(pid, 0);
return false;
} catch (err) {
return (err as NodeJS.ErrnoException).code === "ESRCH";
}
}
private read(): string | null { private read(): string | null {
try { try {
const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, ""); const token = readFileSync(this.opts.file, "utf8").replace(/\n$/, "");
@@ -235,9 +321,10 @@ export class MintedToken implements TokenSource {
} }
} }
/** The scopes as kept beside the token: sorted, one line. */ /** The scopes as kept beside the token: sorted, one line, and whose it is when it is not the admin's. */
private scopesLine(): string { private scopesLine(): string {
return [...this.scopes].sort().join(","); const scopes = [...this.scopes].sort().join(",");
return this.account === this.opts.admin ? scopes : `${scopes} for ${this.account}`;
} }
/** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */ /** The scopes the kept token was minted with, "" when not kept (a token kept before they were). */
@@ -260,17 +347,20 @@ export class MintedToken implements TokenSource {
renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`); renameSync(`${this.opts.file}.scopes.tmp`, `${this.opts.file}.scopes`);
} }
private async doMint(): Promise<string> { /** One call to the forge's API with the admin login, the password read now. */
let password: string; private password(): string {
try { try {
password = readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, ""); return readFileSync(this.opts.passwordFile, "utf8").replace(/\n$/, "");
} catch (err) { } catch (err) {
throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`); throw new Error(`cannot read the admin password at ${this.opts.passwordFile}: ${(err as Error).message}`);
} }
}
private adminCall(password = this.password()): AdminCall {
const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64"); const authorization = "Basic " + Buffer.from(`${this.opts.admin}:${password}`).toString("base64");
const tokens = `${this.opts.url.replace(/\/+$/, "")}/api/v1/users/${encodeURIComponent(this.opts.admin)}/tokens`; const api = `${this.opts.url.replace(/\/+$/, "")}/api/v1`;
const call = async (method: string, path = "", body?: unknown): Promise<{ status: number; body: any }> => { const admin: AdminCall = async (method, path, body) => {
const res = await this.fetchImpl(tokens + path, { const res = await this.fetchImpl(api + path, {
method, method,
headers: { "Content-Type": "application/json", Authorization: authorization }, headers: { "Content-Type": "application/json", Authorization: authorization },
...(body === undefined ? {} : { body: JSON.stringify(body) }), ...(body === undefined ? {} : { body: JSON.stringify(body) }),
@@ -282,56 +372,96 @@ export class MintedToken implements TokenSource {
} }
return { status: res.status, body: parsed }; return { status: res.status, body: parsed };
}; };
return admin;
}
/** Where a retirement of the admin's old token that failed is remembered, to be tried again (retireAdmin). */
private get retireFile(): string {
return `${this.opts.file}.retire-admin`;
}
/**
* The admin's own token by this name goes: what the tools used before must not outlive the move (novox/hq ADR
* 0365 §2). Run after the account's token is minted and kept, so a failure here never leaves the tools without
* a token: it is said, remembered beside the kept token, and tried again when a process next reads that token.
*/
private async retireAdmin(admin: AdminCall = this.adminCall()): Promise<void> {
if (this.account === this.opts.admin) return;
try {
const retired = await this.dropNamed(admin, `/users/${encodeURIComponent(this.opts.admin)}/tokens`, this.opts.admin);
if (retired > 0) this.log(`deleted ${retired} token(s) named "${this.name}" the admin account "${this.opts.admin}" held: the tools act as "${this.account}" now`);
rmSync(this.retireFile, { force: true });
} catch (err) {
writeFileSync(this.retireFile, `${new Date().toISOString()}\n`, { mode: 0o600 });
this.log(`FAILED to delete the token(s) named "${this.name}" the admin account "${this.opts.admin}" holds — tried again at the next start: ${(err as Error).message}`);
}
}
private async doMint(): Promise<string> {
const password = this.password();
const admin = this.adminCall(password);
const tokensOf = (account: string) => `/users/${encodeURIComponent(account)}/tokens`;
const call = (method: string, path = "", body?: unknown) => admin(method, tokensOf(this.account) + path, body);
if (this.account !== this.opts.admin) {
// The tools' account and its team first (novox/hq ADR 0365 §2): a failure here fails the mint, tried again
// at the next call.
const me = await admin("GET", "/user");
if (me.status === 401 || me.status === 403) throw new AdminRefused(this.opts.admin, me.status);
await this.opts.setUp?.(password);
}
let res = await call("POST", "", { name: this.name, scopes: this.scopes }); let res = await call("POST", "", { name: this.name, scopes: this.scopes });
if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status); if (res.status === 401 || res.status === 403) throw new AdminRefused(this.opts.admin, res.status);
if (res.status === 400 || res.status === 422) { if (res.status === 400 || res.status === 422) {
// The forge still holds a token by this name whose value we no longer have — the kept file // The forge still holds a token by this name whose value we no longer have — the kept file
// went while the forge's data stayed. It is ours to replace: drop it by name and mint again. // went while the forge's data stayed, or two processes once minted at once and left two. It is
this.log(`the forge already holds a token named "${this.name}" — replacing it`); // ours to replace: find every one by this name in the account's list and delete each by its id
const dropped = await call("DELETE", `/${encodeURIComponent(this.name)}`); // (gitea refuses a delete by a name held twice: "multiple matches for token name"), then mint.
if (dropped.status === 422) { await this.dropNamed(admin, tokensOf(this.account), this.account, true);
// More than one token carries the name (novox/hq issue 739): two processes minted at once, and the forge's
// check that a name is unused is not atomic. The forge refuses a delete by such a name for good, so every
// one of them is dropped by its id — all are ours, and none of their values is kept.
await this.dropEveryNamed(call, dropped);
} else if (dropped.status !== 204 && dropped.status !== 404) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${dropped.status} ${detail(dropped.body)}`);
}
res = await call("POST", "", { name: this.name, scopes: this.scopes }); res = await call("POST", "", { name: this.name, scopes: this.scopes });
} }
if (res.status !== 201 && res.status !== 200) { if (res.status !== 201 && res.status !== 200) {
throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} ${detail(res.body)}`); throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} ${detail(res.body)}`);
} }
const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null; const token = typeof res.body?.sha1 === "string" ? res.body.sha1 : null;
if (!token) throw new Error(`Gitea POST /users/${this.opts.admin}/tokens: ${res.status} but no token in the reply`); if (!token) throw new Error(`Gitea POST /users/${this.account}/tokens: ${res.status} but no token in the reply`);
this.keep(token); this.keep(token);
this.held = token; this.held = token;
this.log(`minted a token for "${this.opts.admin}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`); this.log(`minted a token for "${this.account}" (${this.scopes.join(", ")}), kept at ${this.opts.file}`);
// Only now, with the account's own token kept, does the admin's old one go.
await this.retireAdmin(admin);
return token; return token;
} }
/** Drop, by id, every token of the admin account that carries this name; the forge listed them, page by page. */ /**
private async dropEveryNamed(call: (method: string, path?: string) => Promise<{ status: number; body: any }>, * Delete every token by this name an account holds, each by its id (gitea refuses a delete by a name held twice:
refused: { status: number; body: any }): Promise<void> { * "multiple matches for token name"); answers how many. `replacing` says the line for a mint that found its name
const ids: number[] = []; * taken.
for (let page = 1; page <= 20; page++) { */
const listed = await call("GET", `?page=${page}&limit=50`); private async dropNamed(admin: AdminCall, tokens: string, account: string, replacing = false): Promise<number> {
const held: number[] = [];
for (let page = 1; page <= 100; page++) {
const listed = await admin("GET", `${tokens}?page=${page}&limit=50`);
if (listed.status !== 200 || !Array.isArray(listed.body)) { if (listed.status !== 200 || !Array.isArray(listed.body)) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${this.name}: ${refused.status} ${detail(refused.body)}; ` + throw new Error(`Gitea GET /users/${account}/tokens: ${listed.status} ${detail(listed.body)}`);
`and its tokens could not be listed to drop them one by one: ${listed.status} ${detail(listed.body)}`);
} }
for (const t of listed.body) if (t?.name === this.name && Number.isInteger(t?.id)) ids.push(t.id); if (listed.body.length === 0) break;
if (listed.body.length < 50) break; for (const t of listed.body) if (t?.name === this.name && typeof t.id === "number") held.push(t.id);
} }
this.log(`the forge holds ${ids.length} tokens named "${this.name}" — dropping each by its id`); if (replacing) {
for (const id of ids) { this.log(held.length === 1
const dropped = await call("DELETE", `/${id}`); ? `the forge already holds a token named "${this.name}" — replacing it`
: `the forge holds ${held.length} tokens named "${this.name}" — deleting each by its id and minting one`);
}
for (const id of held) {
const dropped = await admin("DELETE", `${tokens}/${id}`);
if (dropped.status !== 204 && dropped.status !== 404) { if (dropped.status !== 204 && dropped.status !== 404) {
throw new Error(`Gitea DELETE /users/${this.opts.admin}/tokens/${id}: ${dropped.status} ${detail(dropped.body)}`); throw new Error(`Gitea DELETE /users/${account}/tokens/${id} (named ${this.name}): ${dropped.status} ${detail(dropped.body)}`);
} }
} }
return held.length;
} }
} }
+52
View File
@@ -0,0 +1,52 @@
// The forge's tools act as an account of their own, not as the site administrator (novox/hq ADR 0365 §2,
// issue 469).
//
// The administrator's token reached everything the forge holds, so the tools' boundary rested on their own guards
// (guard.ts): a protection edit, say, the forge itself would have taken. Now the tools' token is minted for a plain
// account, `mesh-tools`, which is no administrator and writes to the mesh's repositories through one team of the
// mesh's organisation:
//
// - **the account**: made by the module with the administrator's login (that login sets the forge up and does
// nothing else), never an administrator — one found made an administrator is made a plain account again. Its
// password is random, sent once and kept nowhere: the account is reached only by the tokens the administrator
// mints for it (gitea lets an administrator mint for another account: `reqSelfOrAdmin`).
// - **the team**: `mesh-tools` in the organisation `novox`, on every repository of it, writing code (pushes to a
// branch, commit statuses, merges), issues (notes, comments, labels) and pull requests. No administration of a
// repository, so the forge refuses the account a protection edit, a repository's settings and its deletion;
// no repository made in the organisation, since its maker would be that repository's administrator.
//
// Every step is idempotent (GiteaAdmin's, client.ts), and run with the administrator's login before each mint of the tools' token, so a team
// or a membership lost behind the module's back is made again at the next mint.
/** The account the tools act as. */
export const TOOLS_ACCOUNT = "mesh-tools";
/** The organisation whose repositories are the mesh's, and the team through which the tools' account writes there. */
export const TOOLS_ORG = "novox";
export const TOOLS_TEAM = "mesh-tools";
/** What the team grants: write on code, issues and pull requests of every repository of the organisation, and no more. */
export const TOOLS_TEAM_UNITS = {
permission: "write",
units_map: { "repo.code": "write", "repo.issues": "write", "repo.pulls": "write" },
includes_all_repositories: true,
can_create_org_repo: false,
} as const;
/** One call to the forge's API with the administrator's login: a path under /api/v1, the status and body answered. */
export type AdminCall = (method: string, path: string, body?: unknown) => Promise<{ status: number; body: any }>;
/** What setting the account up needs of the forge's admin API: GiteaAdmin's own (client.ts), shared with the
* package provisioner, so a create race (422) and a user made by hand are handled once. */
export interface ToolsForge {
ensureUser(username: string, password: string, email: string, plain?: boolean): Promise<void>;
ensureTeamWith(org: string, team: string, units: Record<string, unknown>): Promise<number>;
addUserToTeam(teamId: number, username: string): Promise<void>;
}
/** Make the tools' account, plain, a member of its team on every repository of the organisation. Its password is
* set afresh each time and kept nowhere: the account is reached only by the tokens the administrator mints. */
export async function setUpToolsAccount(forge: ToolsForge, account = TOOLS_ACCOUNT, password: () => string): Promise<void> {
await forge.ensureUser(account, password(), `${account}@noreply.localhost`, true);
const team = await forge.ensureTeamWith(TOOLS_ORG, TOOLS_TEAM, { name: TOOLS_TEAM, ...TOOLS_TEAM_UNITS });
await forge.addUserToTeam(team, account);
}
+17 -10
View File
@@ -10,7 +10,7 @@
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools"; import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { emit } from "@novox/mesh-sdk/events"; import { emit } from "@novox/mesh-sdk/events";
import { GiteaClient, readyAuthors, type BranchProtection } from "../client.js"; import { GiteaAdmin, GiteaClient, readyAuthors, type BranchProtection } from "../client.js";
import { appendNote, run, viewBody, viewComment } from "../delivery.js"; import { appendNote, run, viewBody, viewComment } from "../delivery.js";
import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js"; import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js";
import { import {
@@ -59,13 +59,13 @@ function namesChecked(tools: ToolDefinition[]): ToolDefinition[] {
})); }));
} }
export function getGiteaTools(gitea: GiteaClient): ToolDefinition[] { export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
return namesChecked(allGiteaTools(gitea)); return namesChecked(allGiteaTools(gitea, admin));
} }
// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one // gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one
// writer (novox/hq issue 456, ADR 0323). // writer (novox/hq issue 456, ADR 0323).
function allGiteaTools(gitea: GiteaClient): ToolDefinition[] { function allGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] {
return [ return [
// ---- Repositories ---- // ---- Repositories ----
{ {
@@ -401,8 +401,11 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
}, },
run: async (args) => { run: async (args) => {
const owner = String(args.owner), repo = String(args.repo); const owner = String(args.owner), repo = String(args.repo);
if (!args.branch) return { rules: (await gitea.branchProtections(owner, repo)).map(summarised) }; // The one read made as the forge's administrator (novox/hq ADR 0375, issue 772): the rules' list is the
const rule = await gitea.branchProtection(owner, repo, String(args.branch)); // repository administrator's, which the tools' account is not. No other tool uses this account.
const rules = await (admin ? admin() : gitea).branchProtections(owner, repo);
if (!args.branch) return { rules: rules.map(summarised) };
const rule = rules.find((p) => (p.rule_name ?? p.branch_name) === String(args.branch)) ?? null;
return { branch: String(args.branch), rule: rule ? summarised(rule) : null }; return { branch: String(args.branch), rule: rule ? summarised(rule) : null };
}, },
}, },
@@ -504,7 +507,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
base = base || "main"; base = base || "main";
let rule: BranchProtection | null = null, unreadable: string | undefined; let rule: BranchProtection | null = null, unreadable: string | undefined;
try { try {
rule = await gitea.branchProtection(owner, repo, base); rule = await gitea.appliedProtection(owner, repo, base);
} catch (err) { } catch (err) {
unreadable = err instanceof Error ? err.message : String(err); unreadable = err instanceof Error ? err.message : String(err);
} }
@@ -589,8 +592,12 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
const { owner, repo, branch, newBranch } = judged.contents; const { owner, repo, branch, newBranch } = judged.contents;
const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`); const found = await gitea.api<{ default_branch?: string }>(`/repos/${checkName("owner", owner)}/${checkName("name", repo)}`);
const defaultBranch = String(found?.default_branch ?? "main"); const defaultBranch = String(found?.default_branch ?? "main");
const refused = judgeContentsWrite(newBranch ?? branch ?? defaultBranch, defaultBranch, // The protection the forge applies to the branch written, read from the branch: the rules' own list is a
await gitea.branchProtections(owner, repo)); // repository administrator's, which the tools' account is not (novox/hq ADR 0365). A branch not made yet is
// left to the forge, which refuses a write that would make a branch a rule guards.
const target = newBranch ?? branch ?? defaultBranch;
const applied = target === defaultBranch ? null : await gitea.appliedProtection(owner, repo, target);
const refused = judgeContentsWrite(target, defaultBranch, applied ? [applied] : []);
if (refused) throw new Refused(refused); if (refused) throw new Refused(refused);
} }
// The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts). // The path judged is the path sent: normalised, with only the allow-listed query keys (guard.ts).
@@ -611,7 +618,7 @@ function allGiteaTools(gitea: GiteaClient): ToolDefinition[] {
// way, gitea contributes none rather than failing the whole runtime, and says why. // way, gitea contributes none rather than failing the whole runtime, and says why.
registerModuleTools("gitea", (env) => { registerModuleTools("gitea", (env) => {
try { try {
return getGiteaTools(GiteaClient.fromEnv(env)); return getGiteaTools(GiteaClient.fromEnv(env), () => GiteaAdmin.fromEnv(env));
} catch (err) { } catch (err) {
console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`); console.log(`[gitea] no tools — ${err instanceof Error ? err.message : String(err)}`);
return []; return [];
+241 -36
View File
@@ -140,8 +140,11 @@ export interface ReceiverOptions {
publish: (event: ForgeEvent) => Promise<void>; publish: (event: ForgeEvent) => Promise<void>;
/** Acts on it at once, beside publishing it (the announcer). */ /** Acts on it at once, beside publishing it (the announcer). */
act?: (event: ForgeEvent) => Promise<void>; act?: (event: ForgeEvent) => Promise<void>;
/** Told of every verified delivery, so the polls know events are heard. */ /** Told of every verified delivery — its kind (X-Gitea-Event) and its delivery id — so the polls know events are
heard?: () => void; * heard, and the module can say when it last heard one. */
heard?: (kind: string, delivery: string) => void;
/** Told of every refused delivery, with the reason and where it came from. */
refused?: (reason: string, from: string) => void;
log?: (line: string) => void; log?: (line: string) => void;
} }
@@ -158,6 +161,10 @@ export const REMEMBERED = 2000;
*/ */
export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } { export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void> } {
const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`)); const log = opts.log ?? ((l) => console.log(`[gitea] ${l}`));
const refuse = (from: string, reason: string): void => {
log(`refused a forge event from ${from}: ${reason}`);
opts.refused?.(reason, from);
};
const seen = new Set<string>(); const seen = new Set<string>();
let queue: Promise<void> = Promise.resolve(); let queue: Promise<void> = Promise.resolve();
const server = createServer((req, res) => { const server = createServer((req, res) => {
@@ -168,14 +175,14 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
const from = req.socket.remoteAddress ?? "?"; const from = req.socket.remoteAddress ?? "?";
const signature = String(req.headers["x-gitea-signature"] ?? "").trim(); const signature = String(req.headers["x-gitea-signature"] ?? "").trim();
if (!signature) { if (!signature) {
log(`refused a forge event from ${from}: it is unsigned`); refuse(from, "it is unsigned");
res.writeHead(401, { Connection: "close" }); res.writeHead(401, { Connection: "close" });
res.end("unsigned\n"); res.end("unsigned\n");
req.destroy(); req.destroy();
return; return;
} }
if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) { if (Number(req.headers["content-length"] ?? 0) > MAX_BODY) {
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); refuse(from, `its body is over ${MAX_BODY} bytes`);
res.writeHead(413, { Connection: "close" }); res.writeHead(413, { Connection: "close" });
res.end("too large\n"); res.end("too large\n");
req.destroy(); req.destroy();
@@ -189,7 +196,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
if (size > MAX_BODY) { if (size > MAX_BODY) {
if (!over) { if (!over) {
over = true; over = true;
log(`refused a forge event from ${from}: its body is over ${MAX_BODY} bytes`); refuse(from, `its body is over ${MAX_BODY} bytes`);
res.writeHead(413, { Connection: "close" }); res.writeHead(413, { Connection: "close" });
res.end("too large\n"); res.end("too large\n");
req.destroy(); req.destroy();
@@ -200,7 +207,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
if (over) return; if (over) return;
const body = Buffer.concat(chunks); const body = Buffer.concat(chunks);
if (!verified(opts.secret, body, signature)) { if (!verified(opts.secret, body, signature)) {
log(`refused a forge event from ${from}: its signature does not verify`); refuse(from, "its signature does not verify");
res.statusCode = 401; res.statusCode = 401;
return res.end("signature does not verify\n"); return res.end("signature does not verify\n");
} }
@@ -208,11 +215,13 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
try { try {
payload = JSON.parse(body.toString("utf8")); payload = JSON.parse(body.toString("utf8"));
} catch { } catch {
refuse(from, "it is signed but not JSON");
res.statusCode = 400; res.statusCode = 400;
return res.end("not JSON\n"); return res.end("not JSON\n");
} }
opts.heard?.();
const delivery = String(req.headers["x-gitea-delivery"] ?? ""); const delivery = String(req.headers["x-gitea-delivery"] ?? "");
const kind = String(req.headers["x-gitea-event"] ?? "other");
opts.heard?.(kind, delivery);
if (delivery && seen.has(delivery)) { if (delivery && seen.has(delivery)) {
res.statusCode = 200; res.statusCode = 200;
return res.end("already heard\n"); return res.end("already heard\n");
@@ -221,7 +230,7 @@ export function receiver(opts: ReceiverOptions): Server & { idle(): Promise<void
seen.add(delivery); seen.add(delivery);
if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string); if (seen.size > REMEMBERED) seen.delete(seen.values().next().value as string);
} }
const event = eventOf(String(req.headers["x-gitea-event"] ?? "other"), payload); const event = eventOf(kind, payload);
res.statusCode = 200; res.statusCode = 200;
res.end("queued\n"); res.end("queued\n");
queue = queue.then(async () => { queue = queue.then(async () => {
@@ -242,27 +251,113 @@ export interface HookAdmin {
call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>; call(method: string, path: string, body?: unknown): Promise<{ status: number; body: any }>;
} }
/** The forge's one system webhook as ensured at start: read back from the forge after it was made. */
export interface HookState {
/** Always "system": a hook the forge's listing of system webhooks names, so it fires for every repository. */
kind: "system";
id: number;
url: string;
events: string[];
active: boolean;
content_type?: string;
/** Made where none was, or made in place of the ones there (system, default, or a repository's) at the receiver's url. */
did: "created" | "replaced";
/** The hooks at the receiver's url it took the place of, by kind: a repository's is a copy of a default webhook, which
* would have sent that repository's events a second time. */
removed: { system: number; default: number; repository: number; repositories: string[] };
}
/** How long without a verified delivery — since the start, or since the last one — a forge is said silent once the poll
* finds what no event said. */
export const SILENT_AFTER = 10 * 60_000;
/** /**
* **How fast the polls look** (issue 729): every five minutes only while the forge's events are heard — a verified * **How fast the polls look, and whether the forge's events are heard** (issue 729): every five minutes only while the
* delivery within the last ten minutes, and none missed since. Until the first delivery, after ten minutes without one, * forge's events are heard — a verified delivery within the last ten minutes, and none missed since. Until the first
* or once the poll found something no event said, every thirty seconds. * delivery, after ten minutes without one, or once the poll found something no event said, every thirty seconds.
*
* It also keeps what `gitea_events` says: where the receiver listens, the hook as ensured, the verified and refused
* deliveries, and the polls' finds no event said. **A forge whose events stop is said** (issue 729): once the receiver
* listens and the hook is ensured, when nothing verified for ten minutes — since the start, or since the last delivery —
* and the poll has found at least two heads or merges no event said in that quiet (one could be the poll beating an
* event by a second), it is logged loudly once per silent spell, and the module's events are said unhealthy until a
* delivery verifies. Deliveries refused since the last verified one are said unhealthy too.
*/ */
export class Pace { export class Pace {
private last = 0; private last = 0;
private missed = 0; private missed = 0;
private readonly started: number;
private readonly now: () => number; private readonly now: () => number;
private readonly log: (l: string) => void; private readonly log: (l: string) => void;
private where: { host: string; port: number; url: string } | null = null;
private hook: HookState | null = null;
private notHeard: string | null = null;
private notHeardFails = false;
private verifiedCount = 0;
private lastVerified: { at: number; event: string; delivery: string } | null = null;
private refusedCount = 0;
private refusedSpell = 0;
private lastRefused: { at: number; reason: string; from: string } | null = null;
private missCount = 0;
private lastMiss: { at: number; what: string } | null = null;
private silentSince = 0;
private quietMisses = 0;
constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) { constructor(now: () => number = Date.now, log: (l: string) => void = (l) => console.error(`[gitea] ${l}`)) {
this.now = now; this.now = now;
this.log = log; this.log = log;
this.started = now();
} }
heard(): void { /** The receiver listens, and the hook at it was ensured: from now on, deliveries are expected. */
listening(host: string, port: number, url: string, hook: HookState): void {
this.where = { host, port, url };
this.hook = hook;
this.notHeard = null;
}
/** No events are listened for, and why: by configuration (`fails` false), or because the receiver could not listen
* or the hook could not be ensured (`fails` true, said unhealthy). The polls alone watch. */
unheard(why: string, fails = true): void {
this.notHeard = why;
this.notHeardFails = fails;
}
heard(event = "", delivery = ""): void {
this.last = this.now(); this.last = this.now();
this.verifiedCount++;
this.lastVerified = { at: this.last, event, delivery };
this.refusedSpell = 0;
this.quietMisses = 0;
if (this.silentSince) this.log(`the forge's events are heard again: ${event || "a delivery"} verified`);
this.silentSince = 0;
}
refused(reason: string, from: string): void {
this.refusedCount++;
this.refusedSpell++;
this.lastRefused = { at: this.now(), reason, from };
} }
/** The poll found a head or merge no forge event announced. */ /** The poll found a head or merge no forge event announced. */
miss(what: string): void { miss(what: string): void {
const t = this.now();
if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`); if (this.hearing()) this.log(`THE FORGE'S EVENTS MISSED ${what}: the poll found it, so the polls look every 30 s again until the next event`);
this.missed = this.now(); this.missed = t;
this.missCount++;
this.lastMiss = { at: t, what };
// Silent: nothing verified for SILENT_AFTER — never since the start, or not since the last delivery (a hook deleted by
// hand, every delivery refused) — while the poll finds what no event said. Said once per silent spell.
const quietSince = this.last || this.started;
this.quietMisses++;
if (this.expecting() && t - quietSince >= SILENT_AFTER && this.quietMisses >= 2 && !this.silentSince) {
this.silentSince = t;
const minutes = Math.round((t - quietSince) / 60_000);
const refusedSince = this.refusedSinceHeard();
this.log(`THE FORGE'S EVENTS ARE NOT HEARD: ${this.last ? `no delivery verified since the last at ` +
`${new Date(this.last).toISOString()}` : "no delivery ever verified since the start"}, ${minutes} min, from the ` +
`system webhook at ${this.hook?.url}${refusedSince ? `; ${refusedSince} refused since, the last because ` +
`${this.lastRefused?.reason}` : ""}; the poll found ${what} — the polls alone watch the forge, every 30 s ` +
`(novox/hq issue 729)`);
}
}
/** Whether a delivery was refused after the last verified one (or with none verified). */
private refusedSinceHeard(): number {
return this.lastRefused && this.lastRefused.at >= this.last ? this.refusedSpell : 0;
} }
hearing(): boolean { hearing(): boolean {
const t = this.now(); const t = this.now();
@@ -271,36 +366,146 @@ export class Pace {
every(): number { every(): number {
return this.hearing() ? 5 * 60_000 : 30_000; return this.hearing() ? 5 * 60_000 : 30_000;
} }
private expecting(): boolean {
return this.where !== null && this.hook !== null;
}
/** What `gitea_events` answers: healthy, unhealthy or waits (ADR 0283's shape), with every fact behind it. */
report(): Record<string, unknown> {
const t = this.now();
const iso = (ms: number): string => new Date(ms).toISOString();
let state: "healthy" | "unhealthy" | "waits" = "healthy";
let why: string;
const refusedSince = this.refusedSinceHeard();
if (this.silentSince) {
state = "unhealthy";
why = `${this.last ? `no delivery verified since the last at ${iso(this.last)}` : `the forge has never delivered an ` +
`event since the start at ${iso(this.started)}`}, while the poll found ${this.missCount} head(s) or merge(s) no ` +
`event said, the last ${this.lastMiss?.what} at ${iso(this.lastMiss?.at ?? t)}`;
} else if (refusedSince) {
state = "unhealthy";
why = `${refusedSince} deliveries refused ${this.last ? `since the last verified one at ${iso(this.last)}` :
"and none verified"}, the last at ${iso(this.lastRefused?.at ?? t)} because ${this.lastRefused?.reason}`;
} else if (!this.expecting()) {
why = this.notHeard ? `no events are listened for, so the polls alone watch the forge: ${this.notHeard}` :
"the receiver is not listening yet";
if (this.notHeard && this.notHeardFails) state = "unhealthy";
} else if (this.hearing()) {
why = `the forge's events are heard: ${this.verifiedCount} verified, the last at ${iso(this.last)}`;
} else if (this.last === 0) {
why = `no delivery heard yet; the forge is said silent if the poll finds what no event said ${SILENT_AFTER / 60_000} min after the start`;
} else {
why = `the last delivery was at ${iso(this.last)}; the polls look every 30 s until the next`;
}
return {
healthy: state !== "unhealthy", state, why,
started: iso(this.started),
receiver: this.where ? { listening: true, ...this.where } : { listening: false, why: this.notHeard ?? "not started" },
hook: this.hook,
verified: { count: this.verifiedCount, last: this.lastVerified && { ...this.lastVerified, at: iso(this.lastVerified.at) } },
refused: { count: this.refusedCount, since_verified: refusedSince, last: this.lastRefused && { ...this.lastRefused, at: iso(this.lastRefused.at) } },
polls: { every_seconds: this.every() / 1000, hearing: this.hearing(), found_unsaid: this.missCount,
last_found: this.lastMiss && { ...this.lastMiss, at: iso(this.lastMiss.at) } },
silent: this.silentSince ? { since: iso(this.silentSince) } : null,
};
}
} }
/** Every event family the forge (1.27) can send a webhook. */ /** Every event family the forge (1.27) can send a webhook. */
export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request", export const HOOK_EVENTS = ["create", "delete", "fork", "push", "issues", "issue_comment", "pull_request",
"pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"]; "pull_request_review_request", "wiki", "repository", "release", "package", "status", "workflow_run", "workflow_job"];
/** async function hooksAt(admin: HookAdmin, type: "system" | "default", url: string): Promise<any[]> {
* **One system webhook, for every event, at the receiver** (issue 729): created when none points there, corrected const listed = await admin.call("GET", `/admin/hooks?type=${type}&limit=50`);
* when one does but differs (events, active, content type), and its secret set again each start — the forge never if (listed.status !== 200) throw new Error(`the forge's ${type} webhooks could not be listed: ${listed.status}`);
* hands a secret back, so a corrected secret cannot be told from a kept one. Answers what it did. return (Array.isArray(listed.body) ? listed.body : []).filter((h) => h?.config?.url === url);
*/ }
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<"created" | "corrected"> {
// POST /admin/hooks makes a system webhook, and its options take no flag for it (the forge's 1.27.3 API schema). The /** Every page of a listing that answers an array (or `{ data: [] }`, the repository search), PAGE at a time. */
// forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it. const PAGE = 50;
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*", async function everyPage(admin: HookAdmin, path: string, what: string): Promise<any[]> {
config: { url, content_type: "json", secret } }; const all: any[] = [];
const listed = await admin.call("GET", "/admin/hooks?type=system&limit=50"); for (let page = 1; ; page++) {
if (listed.status !== 200) throw new Error(`the forge's system webhooks could not be listed: ${listed.status}`); const r = await admin.call("GET", `${path}${path.includes("?") ? "&" : "?"}page=${page}&limit=${PAGE}`);
const hooks: any[] = Array.isArray(listed.body) ? listed.body : []; if (r.status !== 200) throw new Error(`${what} could not be listed: ${r.status}`);
const ours = hooks.filter((h) => h?.config?.url === url); const items: any[] = Array.isArray(r.body) ? r.body : Array.isArray(r.body?.data) ? r.body.data : [];
if (ours.length === 0) { all.push(...items);
const made = await admin.call("POST", "/admin/hooks", want); if (items.length < PAGE) return all;
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
return "created";
} }
for (const extra of ours.slice(1)) await admin.call("DELETE", `/admin/hooks/${extra.id}`); }
const fixed = await admin.call("PATCH", `/admin/hooks/${ours[0].id}`, { active: true, events: HOOK_EVENTS, branch_filter: "*",
config: { url, content_type: "json", secret } }); /** Every repository's own webhooks at the receiver's url: copies of a default webhook made at it. */
if (fixed.status !== 200) throw new Error(`the forge's system webhook could not be corrected: ${fixed.status}`); async function repoHooksAt(admin: HookAdmin, url: string): Promise<{ repo: string; id: number }[]> {
return "corrected"; const found: { repo: string; id: number }[] = [];
for (const repo of await everyPage(admin, "/repos/search?sort=id", "the forge's repositories")) {
const full = String(repo?.full_name ?? "");
if (!full.includes("/")) continue;
const hooks = await everyPage(admin, `/repos/${full}/hooks`, `${full}'s webhooks`);
for (const h of hooks) if (h?.config?.url === url) found.push({ repo: full, id: Number(h.id) });
}
return found;
}
/**
* **One system webhook, for every event, at the receiver** (issue 729), made each start and read back.
*
* The forge (1.27.3, routers/api/v1/utils/hook.go addHook) makes a *default* webhook from POST /admin/hooks unless
* `config.is_system_webhook` is "true": a default webhook fires for no repository and is only copied into repositories
* made after it, and the listing of system webhooks does not name it — so without the flag every start made another
* one and no event came. Its edit (editHook) never changes a webhook's secret nor its kind, so a hook already there is
* not corrected but replaced: the new system webhook is made first, with the secret kept now, so no event falls in a
* gap; then every other hook at the receiver's url is deleted — system, default, and each repository's copy of a
* default one, which would send that repository's events twice. The forge's listing of system webhooks must then name
* exactly one at the url, or this fails, saying so.
*/
export async function ensureSystemHook(admin: HookAdmin, url: string, secret: string): Promise<HookState> {
// The forge has no "every event" word: each family is named, and a family name (issues, pull_request) takes all of it.
const want = { type: "gitea", active: true, events: HOOK_EVENTS, branch_filter: "*",
config: { url, content_type: "json", secret, is_system_webhook: "true" } };
const before = new Set((await hooksAt(admin, "system", url)).map((h) => Number(h.id)));
const made = await admin.call("POST", "/admin/hooks", want);
if (made.status !== 201 && made.status !== 200) throw new Error(`the forge's system webhook could not be made: ${made.status}`);
const fresh = (await hooksAt(admin, "system", url)).filter((h) => !before.has(Number(h.id)));
if (fresh.length !== 1) {
throw new Error(`the forge names ${fresh.length} new system webhooks at ${url} after one was made, not one: its events ` +
`are not known to be sent`);
}
const ours = fresh[0];
const defaults = await hooksAt(admin, "default", url);
const gone = async (path: string): Promise<void> => {
const r = await admin.call("DELETE", path);
if (r.status !== 204 && r.status !== 200 && r.status !== 404) throw new Error(`the forge's webhook ${path} could not be deleted: ${r.status}`);
};
for (const id of before) await gone(`/admin/hooks/${id}`);
for (const h of defaults) await gone(`/admin/hooks/${h.id}`);
const copies = await repoHooksAt(admin, url);
for (const c of copies) await gone(`/repos/${c.repo}/hooks/${c.id}`);
const kept = await hooksAt(admin, "system", url);
if (kept.length !== 1 || Number(kept[0].id) !== Number(ours.id)) {
throw new Error(`the forge names ${kept.length} system webhooks at ${url} once the others were deleted, not the one ` +
`made (${ours.id})`);
}
const removed = { system: before.size, default: defaults.length, repository: copies.length,
repositories: [...new Set(copies.map((c) => c.repo))] };
return { kind: "system", id: Number(ours.id), url, events: Array.isArray(ours.events) ? ours.events : [],
active: !!ours.active, content_type: ours.config?.content_type,
did: removed.system + removed.default + removed.repository > 0 ? "replaced" : "created", removed };
}
/** The `gitea_events` tool: what the module knows of the forge's events, from its Pace. */
export function eventsTool(pace: Pace): { name: string; description: string; input: Record<string, unknown>;
run: (args: Readonly<Record<string, unknown>>) => Promise<unknown> } {
return {
name: "gitea_events",
description: "Whether the forge's events reach the mesh (novox/hq issue 729): where the receiver listens, the " +
"system webhook as ensured at the start (its id, url, events, kind, and what it replaced), how many deliveries " +
"verified and the last one's time and event, how many were refused and the last reason, how often the polls look " +
"and whether events are heard, and how many heads or merges the poll found that no event said. Unhealthy when the " +
"forge has delivered nothing that verified for ten minutes (since the start or since the last delivery) while the " +
"poll found what no event said, when deliveries were refused since the last verified one, or when the receiver " +
"could not listen or the webhook could not be made.",
input: {},
run: async () => pace.report(),
};
} }
/** /**
+52
View File
@@ -0,0 +1,52 @@
# kernel
The kernel's run-time settings on a machine (novox/hq to-be 42 Phase 1, row 11). Today it holds one:
**transparent huge pages**, set per machine. The rest of the row (the kernel package, microcode, boot
entries) is not built yet.
## Why (novox/hq issue 746)
With transparent huge pages `always`, the page cache of a file mapped by a program is collapsed into
2 MiB folios (khugepaged), and a small write into such a folio dirties all of it: writeback then writes
2 MiB. journald writes its journal files through mmap, and every journal reader (fail2ban's journal
jails among them) maps them too. Measured on the control node on 2026-10-14: journald's cgroup had
written 2.5 TB in 12 days while the journal grew by about 25-30 KB/s, fail2ban's cgroup 63 GB in 4 h
while its own files did not grow, both holding hundreds of MiB of file pages in huge folios; the bursts
saturated the mirrored spinning root disk that holds the bus store and the store, and the controller
lost its lease four times in 30 minutes. With `madvise`, only memory a program asks to have in huge pages
(`madvise(MADV_HUGEPAGE)`) gets them; a database or a JVM that wants them still asks.
## What it owns
| | what | notes |
|---|---|---|
| setting | `transparent-huge-pages` | `always`, `madvise` or `never`. Default `madvise`, the value the mesh chose for every node (ADR 0381), not what the distribution's kernel boots with (`always`); a machine that needs another value is given it as its own setting, with its reason |
| file | `${dir:state}/huge-pages` | the script, written from the setting: `apply` writes the mode to `/sys/kernel/mm/transparent_hugepage/enabled` and reads it back (a value that does not take fails the unit); `restore` gives back the mode found before the module first set it, kept in `${dir:state}/found` |
| file | `/etc/systemd/system/mesh-kernel-huge-pages.service` | oneshot, early in boot (`sysinit.target`, before the services that fill the page cache); reloading it runs `apply`, stopping it runs `restore`, which also removes `found`. Sandboxed: the file system read-only but for the module's state (`/sys` stays writable), no home, no network, no new privileges; the journal's rotation goes over its local socket, which a private network leaves reachable |
| service | `mesh-kernel-huge-pages.service` | running and enabled; **reloaded** when the script (so the setting) changes, so a new value is set directly and never passes back through the found one; restarted only when the unit changes; its health is the unit's |
The per-size controls (`hugepages-<n>kB/enabled`) are left alone: on the machines read, 2048 kB says
`inherit` (it follows the mode) and every other size `never`. `kernel_huge_pages` shows each one.
**A folio already in the page cache keeps its size.** A new mode applies to pages faulted or collapsed
from then on. The active journal files would keep their 2 MiB folios until archived, so `apply` rotates
the journal (`journalctl --rotate`) whenever it moves the mode away from `always`: new files are mapped
under the new mode. A failed rotation is said in the unit's journal and does not fail the setting. No
rotation is made while the machine is still starting (`systemctl is-system-running` says `initializing`
or `starting`): nothing is in huge pages yet, and the persistent journal opens after this unit.
## Tools
| tool | | answers |
|---|---|---|
| `kernel_huge_pages` | r | the mode, defrag and shared-memory mode; each per-size control and what `inherit` resolves to; huge pages by kind from `/proc/meminfo` (MiB); the units holding the most file pages in huge folios (`memory.stat` `file_thp`) with their dirty and writeback bytes, khugepaged's collapses and the bytes their cgroup wrote (`io.stat`, the most any one device saw: a mirror and its members each count the same write) |
## Tests
`go test ./...`: the tool over a stand-in tree holding the control node's own files of 2026-10-14; the
manifest's shape (the default, the order, the unit and its sandbox, reload, restart and health); and the script as the mesh
writes it, run with `sh` against a stand-in directory: it sets and reads back, keeps what it found once,
rotates the journal once and not for `always` nor while the machine starts, goes straight to a new value on a
reload without rewriting what it found, gives back what it found and then keeps nothing, refuses a value the kernel
does not take before writing anything, fails when the write does not take, and says when the kernel has
no transparent huge pages.
@@ -0,0 +1,241 @@
package main
import (
"fmt"
"os"
"path"
"sort"
"strconv"
"strings"
)
// THPDir is where the kernel says and takes its transparent huge pages.
const THPDir = "sys/kernel/mm/transparent_hugepage"
// CgroupRoot is the unified cgroup hierarchy.
const CgroupRoot = "sys/fs/cgroup"
// HugePagesAnswer is what kernel_huge_pages answers.
type HugePagesAnswer struct {
// Mode is the word the kernel brackets in enabled: always, madvise or never.
Mode string `json:"mode"`
Defrag string `json:"defrag,omitempty"`
Shmem string `json:"shmem_enabled,omitempty"`
PerSize []Size `json:"per_size,omitempty"`
// Meminfo is the machine's huge pages by kind, in MiB.
Meminfo map[string]float64 `json:"meminfo_mib,omitempty"`
Units []Unit `json:"units"`
Said []string `json:"said"`
}
// Size is one per-size control (hugepages-<n>kB/enabled).
type Size struct {
Size string `json:"size"`
Enabled string `json:"enabled"`
// Means is what inherit resolves to: the top-level mode.
Means string `json:"means"`
}
// Unit is one cgroup holding file pages in huge folios.
type Unit struct {
Cgroup string `json:"cgroup"`
FileTHP int64 `json:"file_thp_bytes"`
FileDirty int64 `json:"file_dirty_bytes"`
FileWriteback int64 `json:"file_writeback_bytes"`
FileMapped int64 `json:"file_mapped_bytes"`
CollapsedByKhug int64 `json:"thp_collapse_alloc"`
Written int64 `json:"written_bytes"`
}
// bracketed is the word a sysfs choice file brackets: "always [madvise] never" is madvise.
func bracketed(text string) string {
open := strings.IndexByte(text, '[')
end := strings.IndexByte(text, ']')
if open < 0 || end < open {
return ""
}
return text[open+1 : end]
}
// HugePages reads the mode, the per-size controls, meminfo and the top units by file_thp.
func (m *Machine) HugePages(units int) (*HugePagesAnswer, error) {
enabled, err := m.read(THPDir + "/enabled")
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("this kernel has no /%s/enabled: it was built without transparent huge pages", THPDir)
}
return nil, fmt.Errorf("reading /%s/enabled: %w", THPDir, err)
}
a := &HugePagesAnswer{Mode: bracketed(enabled), Units: []Unit{}}
if s, err := m.read(THPDir + "/defrag"); err == nil {
a.Defrag = bracketed(s)
}
if s, err := m.read(THPDir + "/shmem_enabled"); err == nil {
a.Shmem = bracketed(s)
}
a.Said = append(a.Said, fmt.Sprintf("transparent huge pages: %s, defrag %s", a.Mode, orUnknown(a.Defrag)))
sizes, _ := m.dirs(THPDir)
for _, d := range sizes {
if !strings.HasPrefix(d, "hugepages-") {
continue
}
s, err := m.read(THPDir + "/" + d + "/enabled")
if err != nil {
continue
}
sz := Size{Size: strings.TrimPrefix(d, "hugepages-"), Enabled: bracketed(s)}
sz.Means = sz.Enabled
if sz.Enabled == "inherit" {
sz.Means = a.Mode
}
a.PerSize = append(a.PerSize, sz)
}
sort.Slice(a.PerSize, func(i, j int) bool { return kib(a.PerSize[i].Size) < kib(a.PerSize[j].Size) })
var on []string
for _, s := range a.PerSize {
if s.Means != "never" {
on = append(on, s.Size+" "+s.Means)
}
}
if len(a.PerSize) > 0 {
a.Said = append(a.Said, "sizes not never: "+orNone(on))
}
if mi, err := m.read("proc/meminfo"); err == nil {
a.Meminfo = meminfo(mi)
a.Said = append(a.Said, fmt.Sprintf("huge pages held: anonymous %.0f MiB, shared memory %.0f MiB, file %.0f MiB (mapped at 2 MiB %.0f MiB)",
a.Meminfo["AnonHugePages"], a.Meminfo["ShmemHugePages"], a.Meminfo["FileHugePages"], a.Meminfo["FilePmdMapped"]))
}
if units > 0 {
all := m.cgroups()
sort.Slice(all, func(i, j int) bool {
if all[i].FileTHP != all[j].FileTHP {
return all[i].FileTHP > all[j].FileTHP
}
return all[i].Cgroup < all[j].Cgroup
})
if len(all) > units {
all = all[:units]
}
a.Units = all
for _, u := range all {
a.Said = append(a.Said, fmt.Sprintf("%s: file in huge folios %s, dirty %s, writeback %s, written %s since it started",
u.Cgroup, mib(u.FileTHP), mib(u.FileDirty), mib(u.FileWriteback), mib(u.Written)))
}
if len(all) == 0 {
a.Said = append(a.Said, "no unit holds file pages in huge folios")
}
}
return a, nil
}
// cgroups walks the leaf services and scopes (to three levels below the root) that hold file_thp.
func (m *Machine) cgroups() []Unit {
var out []Unit
var walk func(rel string, depth int)
walk = func(rel string, depth int) {
children, _ := m.dirs(path.Join(CgroupRoot, rel))
for _, c := range children {
p := path.Join(rel, c)
if strings.HasSuffix(c, ".slice") {
if depth < 3 {
walk(p, depth+1)
}
continue
}
if !strings.HasSuffix(c, ".service") && !strings.HasSuffix(c, ".scope") {
continue
}
stat, err := m.read(path.Join(CgroupRoot, p, "memory.stat"))
if err != nil {
continue
}
kv := memoryStat(stat)
if kv["file_thp"] == 0 {
continue
}
u := Unit{Cgroup: p, FileTHP: kv["file_thp"], FileDirty: kv["file_dirty"], FileWriteback: kv["file_writeback"],
FileMapped: kv["file_mapped"], CollapsedByKhug: kv["thp_collapse_alloc"]}
if io, err := m.read(path.Join(CgroupRoot, p, "io.stat")); err == nil {
u.Written = mostWritten(io)
}
out = append(out, u)
}
}
walk("", 0)
return out
}
// memoryStat reads memory.stat's "key value" lines.
func memoryStat(text string) map[string]int64 {
kv := map[string]int64{}
for _, l := range strings.Split(text, "\n") {
f := strings.Fields(l)
if len(f) != 2 {
continue
}
if n, err := strconv.ParseInt(f[1], 10, 64); err == nil {
kv[f[0]] = n
}
}
return kv
}
// mostWritten is the most bytes any one device of io.stat saw written: a mirror's member disks and the
// mirror itself each count the same write, so the lines are never summed.
func mostWritten(text string) int64 {
var most int64
for _, l := range strings.Split(text, "\n") {
for _, f := range strings.Fields(l) {
if v, ok := strings.CutPrefix(f, "wbytes="); ok {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > most {
most = n
}
}
}
}
return most
}
// meminfo reads the huge page lines of /proc/meminfo, in MiB.
func meminfo(text string) map[string]float64 {
want := map[string]bool{"AnonHugePages": true, "ShmemHugePages": true, "ShmemPmdMapped": true, "FileHugePages": true, "FilePmdMapped": true}
out := map[string]float64{}
for _, l := range strings.Split(text, "\n") {
k, v, ok := strings.Cut(l, ":")
if !ok || !want[k] {
continue
}
f := strings.Fields(v)
if len(f) == 0 {
continue
}
if n, err := strconv.ParseFloat(f[0], 64); err == nil {
out[k] = n / 1024
}
}
return out
}
func kib(size string) int {
n, _ := strconv.Atoi(strings.TrimSuffix(size, "kB"))
return n
}
func mib(b int64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1<<20)) }
func orUnknown(s string) string {
if s == "" {
return "unknown"
}
return s
}
func orNone(list []string) string {
if len(list) == 0 {
return "none"
}
return strings.Join(list, ", ")
}
@@ -0,0 +1,113 @@
package main
// Over a stand-in tree holding novox's own files of 2026-10-14 23:25 (read through node-reads for
// novox/hq issue 746): the mode, the per-size controls, meminfo's huge page lines, and fail2ban's and
// journald's memory.stat and io.stat, cut to the lines the tool reads.
import (
"os"
"path/filepath"
"strings"
"testing"
)
func tree(t *testing.T, files map[string]string) *Machine {
t.Helper()
root := t.TempDir()
for p, content := range files {
full := filepath.Join(root, p)
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
return &Machine{Root: root}
}
func novox(t *testing.T) *Machine {
thp := "sys/kernel/mm/transparent_hugepage/"
cg := "sys/fs/cgroup/system.slice/"
files := map[string]string{
thp + "enabled": "[always] madvise never\n",
thp + "defrag": "always defer defer+madvise [madvise] never\n",
thp + "shmem_enabled": "always within_size [advise] never deny force\n",
thp + "hugepages-2048kB/enabled": "always [inherit] madvise never\n",
thp + "hugepages-64kB/enabled": "always inherit madvise [never]\n",
thp + "hugepages-1024kB/enabled": "always inherit madvise [never]\n",
"proc/meminfo": "MemTotal: 65748808 kB\nAnonHugePages: 3579904 kB\nShmemHugePages: 4096 kB\n" +
"ShmemPmdMapped: 4096 kB\nFileHugePages: 1693696 kB\nFilePmdMapped: 882688 kB\nHugepagesize: 2048 kB\n",
cg + "fail2ban.service/memory.stat": "anon 21815296\nfile 204623872\nfile_mapped 130736128\nfile_dirty 27262976\n" +
"file_writeback 4096\nanon_thp 0\nfile_thp 197132288\nthp_fault_alloc 0\nthp_collapse_alloc 550\n",
cg + "fail2ban.service/io.stat": "9:0 \n8:16 rbytes=3223552 wbytes=67723415552 rios=240 wios=281159 dbytes=0 dios=0\n" +
"8:0 rbytes=6152192 wbytes=67723415552 rios=832 wios=281159 dbytes=0 dios=0\n" +
"9:2 rbytes=9375744 wbytes=67723415552 rios=1072 wios=281159 dbytes=0 dios=0\n",
cg + "systemd-journald.service/memory.stat": "anon 3227648\nfile 1032015872\nfile_mapped 131928064\nfile_dirty 622592\n" +
"file_writeback 159744\nanon_thp 2097152\nfile_thp 379584512\nthp_fault_alloc 0\nthp_collapse_alloc 11741\n",
cg + "systemd-journald.service/io.stat": "9:0 \n8:16 rbytes=55209984 wbytes=2498768035840 rios=1317 wios=24719004 dbytes=0 dios=0\n" +
"8:0 rbytes=72953856 wbytes=2498768039936 rios=2194 wios=24719005 dbytes=0 dios=0\n" +
"9:2 rbytes=128163840 wbytes=2498724843520 rios=3511 wios=24707873 dbytes=0 dios=0\n",
cg + "sshd.service/memory.stat": "anon 1000\nfile_thp 0\n",
}
return tree(t, files)
}
func TestNovoxBeforeTheChange(t *testing.T) {
a, err := novox(t).HugePages(10)
if err != nil {
t.Fatal(err)
}
if a.Mode != "always" || a.Defrag != "madvise" || a.Shmem != "advise" {
t.Fatalf("modes: %+v", a)
}
if len(a.PerSize) != 3 || a.PerSize[0].Size != "64kB" || a.PerSize[2].Size != "2048kB" ||
a.PerSize[2].Enabled != "inherit" || a.PerSize[2].Means != "always" || a.PerSize[0].Means != "never" {
t.Fatalf("per size: %+v", a.PerSize)
}
if a.Meminfo["FileHugePages"] != 1654 || a.Meminfo["FilePmdMapped"] != 862 {
t.Fatalf("meminfo: %v", a.Meminfo)
}
if len(a.Units) != 2 || a.Units[0].Cgroup != "system.slice/systemd-journald.service" ||
a.Units[1].Cgroup != "system.slice/fail2ban.service" {
t.Fatalf("units: %+v", a.Units)
}
// The mirror and its two members each count the write: the most any one saw, never their sum.
if a.Units[1].Written != 67723415552 || a.Units[1].FileTHP != 197132288 || a.Units[1].CollapsedByKhug != 550 {
t.Fatalf("fail2ban: %+v", a.Units[1])
}
said := strings.Join(a.Said, "\n")
for _, want := range []string{"transparent huge pages: always, defrag madvise", "sizes not never: 2048kB always",
"file 1654 MiB (mapped at 2 MiB 862 MiB)", "system.slice/fail2ban.service: file in huge folios 188.0 MiB"} {
if !strings.Contains(said, want) {
t.Errorf("said lacks %q:\n%s", want, said)
}
}
}
func TestUnitsAreBoundedAndNoneIsSaid(t *testing.T) {
a, err := novox(t).HugePages(1)
if err != nil || len(a.Units) != 1 {
t.Fatalf("%v %+v", err, a)
}
b, err := tree(t, map[string]string{"sys/kernel/mm/transparent_hugepage/enabled": "always [madvise] never\n"}).HugePages(10)
if err != nil || b.Mode != "madvise" || len(b.Units) != 0 || !strings.Contains(strings.Join(b.Said, "\n"), "no unit holds") {
t.Fatalf("%v %+v", err, b)
}
}
func TestAKernelWithoutHugePagesIsAnError(t *testing.T) {
if _, err := tree(t, map[string]string{"proc/meminfo": ""}).HugePages(10); err == nil ||
!strings.Contains(err.Error(), "built without transparent huge pages") {
t.Fatalf("%v", err)
}
}
func TestTheArgumentIsBounded(t *testing.T) {
if _, err := whole(map[string]any{"units": -1.0}, "units", 10, 0, 100); err == nil {
t.Error("a negative count was accepted")
}
if n, _ := whole(map[string]any{"units": 1000.0}, "units", 10, 0, 100); n != 100 {
t.Errorf("not bounded: %d", n)
}
}
@@ -0,0 +1,65 @@
package main
// The machine as the tools see it: the files they read, under a root that tests replace with a stand-in
// tree. Everything read here is readable by the runtime's account; nothing is escalated.
import (
"fmt"
"io/fs"
"os"
"path/filepath"
)
// Machine reads this machine's kernel files under Root ("/" on the machine).
type Machine struct {
Root string
}
// ThisMachine is the machine the runtime launched this bundle on.
func ThisMachine() *Machine { return &Machine{Root: "/"} }
func (m *Machine) path(p string) string { return filepath.Join(m.Root, p) }
func (m *Machine) read(p string) (string, error) {
b, err := os.ReadFile(m.path(p))
return string(b), err
}
func (m *Machine) dirs(p string) ([]string, error) {
entries, err := os.ReadDir(m.path(p))
if err != nil {
return nil, err
}
var out []string
for _, e := range entries {
if e.IsDir() || e.Type()&fs.ModeSymlink != 0 {
out = append(out, e.Name())
}
}
return out, nil
}
// whole is a whole-number argument with a default, kept within bounds.
func whole(args map[string]any, key string, def, least, most int) (int, error) {
raw, present := args[key]
if !present || raw == nil {
return def, nil
}
f, ok := raw.(float64)
if !ok || f != float64(int(f)) {
return 0, fmt.Errorf("%s must be a whole number", key)
}
n := int(f)
if n < least {
return 0, fmt.Errorf("%s must be at least %d", key, least)
}
if n > most {
n = most
}
return n, nil
}
// schema is a tool's input: its properties.
func schema(properties map[string]any) map[string]any {
return map[string]any{"type": "object", "properties": properties}
}
+50
View File
@@ -0,0 +1,50 @@
// The kernel module's tools bundle (novox/hq to-be 42 Phase 1 row 11, issue 746).
//
// Served by the node's runtime over MCP on stdio through the Go SDK (ADR 0188, ADR 0193). It only reads:
// the transparent huge pages the kernel runs with, and which units hold file pages in huge folios, which
// is what a small write to a mapped file costs a whole huge page of writeback for (issue 746).
package main
import (
"fmt"
"os"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// binaryName is what the build names this bundle's executable: the manifest's `binary`.
const binaryName = "kernel-tools"
func main() {
// An empty name serves as the module the runtime names (MESH_SERVED_MODULE): kernel.
if err := stdio.Serve("", tools(ThisMachine())); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func tools(m *Machine) []stdio.Tool {
return []stdio.Tool{
{
Name: "kernel_huge_pages",
Description: "Transparent huge pages on this machine: the mode in force (always, madvise or never) and defrag, " +
"each per-size control with what inherit resolves to, the machine's huge pages by kind (anonymous, " +
"shared memory, file, file mapped at 2 MiB) from /proc/meminfo, and the units holding the most file " +
"pages in huge folios (memory.stat file_thp) with their dirty and writeback bytes and the bytes their " +
"cgroup has written (io.stat, the most any one device saw). A unit with file_thp that writes far more " +
"than its files grow is paying a whole huge page of writeback per small write (novox/hq issue 746). " +
"Replaces cat /sys/kernel/mm/transparent_hugepage/enabled, grep Huge /proc/meminfo and reading " +
"memory.stat by hand. (r)",
Input: schema(map[string]any{
"units": map[string]any{"type": "integer", "description": "how many units to name, the most file_thp first (default 10, 0 for none, at most 100)"},
}),
Run: func(args map[string]any) (any, error) {
n, err := whole(args, "units", 10, 0, 100)
if err != nil {
return nil, err
}
return m.HugePages(n)
},
},
}
}
@@ -0,0 +1,239 @@
package main
// The module's shape (novox/hq issue 746): the setting's default is madvise, the value the mesh chose
// (ADR 0381), not what the kernel boots with; the script is written into the
// module's own state and run by a sandboxed oneshot unit that is reloaded (applied again) when the script,
// and so the setting, changes and restarted only when the unit does; and the script itself sets, verifies,
// gives back and refuses as it says.
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func TestTheDefaultIsMadviseAndTheUnitFollowsTheSetting(t *testing.T) {
m := manifest(t)
s := m.Settings["transparent-huge-pages"]
if s["kind"] != "preference" || s["default"] != "madvise" || s["why"] == "" {
t.Fatalf("the setting: %v", s)
}
var ids []string
for _, r := range m.Resources {
ids = append(ids, r["id"].(string))
}
if strings.Join(ids, " ") != "state huge-pages unit apply" {
t.Fatalf("order: %v", ids)
}
script := m.resource(t, "huge-pages")
// The script runs as root, so the setting it is written from is set at the terminal alone (issue 339).
if script["path"] != "${dir:state}/huge-pages" || script["mode"] != "0755" || script["trusted"] != true ||
!strings.Contains(script["content"].(string), "want='${setting:transparent-huge-pages}'") {
t.Fatalf("the script: %v", script["path"])
}
unit := m.resource(t, "unit")["content"].(string)
for _, line := range []string{"Type=oneshot", "RemainAfterExit=yes", "ExecStart=${dir:state}/huge-pages apply",
"ExecReload=${dir:state}/huge-pages apply", "ExecStop=${dir:state}/huge-pages restore", "WantedBy=sysinit.target",
"DefaultDependencies=no", "ProtectSystem=strict", "ReadWritePaths=${dir:state}", "ProtectHome=yes", "PrivateNetwork=yes",
"NoNewPrivileges=yes"} {
if !strings.Contains(unit, line+"\n") {
t.Errorf("the unit lacks %q", line)
}
}
apply := m.resource(t, "apply")
if apply["unit"] != "mesh-kernel-huge-pages.service" || apply["state"] != "running" || apply["boot"] != "enabled" {
t.Fatalf("apply: %v", apply)
}
// A new value reloads (apply again); only a new unit restarts, whose stop would pass back through the
// mode the machine was found with.
if on := apply["restart-on"].([]any); len(on) != 1 || on[0] != "unit" {
t.Fatalf("restart-on: %v", on)
}
if on := apply["reload-on"].([]any); len(on) != 1 || on[0] != "huge-pages" {
t.Fatalf("reload-on: %v", on)
}
if h := apply["health"].(map[string]any); h["kind"] != "unit" {
t.Fatalf("health: %v", h)
}
}
// bed is the script as the mesh writes it for one value of the setting, a stand-in transparent_hugepage
// directory, the module's state, and a PATH whose journalctl records that it was asked to rotate.
type bed struct {
script, sys, state, rotated, path, bin string
}
func newBed(t *testing.T, want, found string) bed {
return newBedIn(t, t.TempDir(), want, found)
}
// newBedIn writes the script for want into dir; a second bed in the same dir shares the machine and the
// state, as a reload with a new value does.
func newBedIn(t *testing.T, dir, want, found string) bed {
t.Helper()
b := bed{script: filepath.Join(dir, "huge-pages"), sys: filepath.Join(dir, "thp"), state: filepath.Join(dir, "state"),
rotated: filepath.Join(dir, "rotated")}
content := manifest(t).resource(t, "huge-pages")["content"].(string)
content = strings.ReplaceAll(content, "${setting:transparent-huge-pages}", want)
content = strings.ReplaceAll(content, "${dir:state}", b.state)
if strings.Contains(content, "${") {
t.Fatalf("a substitution the test does not make is left in the script")
}
bin := filepath.Join(dir, "bin")
for _, d := range []string{b.sys, b.state, bin} {
if err := os.MkdirAll(d, 0o755); err != nil {
t.Fatal(err)
}
}
write(t, b.script, content, 0o755)
if found != "" {
write(t, filepath.Join(b.sys, "enabled"), found, 0o644)
}
write(t, filepath.Join(bin, "journalctl"), "#!/bin/sh\necho \"$@\" >>"+b.rotated+"\n", 0o755)
b.bin = bin
b.booted(t, "running")
b.path = bin + string(os.PathListSeparator) + os.Getenv("PATH")
return b
}
// booted makes the service manager say state for is-system-running.
func (b bed) booted(t *testing.T, state string) {
t.Helper()
write(t, filepath.Join(b.bin, "systemctl"), "#!/bin/sh\n[ \"$1\" = is-system-running ] && echo "+state+"\n", 0o755)
}
func write(t *testing.T, p, content string, mode os.FileMode) {
t.Helper()
if err := os.WriteFile(p, []byte(content), mode); err != nil {
t.Fatal(err)
}
}
func (b bed) run(t *testing.T, verb string) (string, error) {
t.Helper()
cmd := exec.Command("sh", b.script, verb, b.sys, b.state)
cmd.Env = append(os.Environ(), "PATH="+b.path)
out, err := cmd.CombinedOutput()
return string(out), err
}
func (b bed) file(t *testing.T, p string) string {
t.Helper()
raw, err := os.ReadFile(p)
if err != nil {
return ""
}
return string(raw)
}
func TestApplySetsTheModeRotatesTheJournalOnceAndRestoreGivesBackWhatWasFound(t *testing.T) {
// novox's file on 2026-10-14, read through node-reads.
b := newBed(t, "madvise", "[always] madvise never\n")
out, err := b.run(t, "apply")
if err != nil || !strings.Contains(out, "always -> madvise") {
t.Fatalf("apply: %v %s", err, out)
}
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
t.Fatalf("written: %q", got)
}
if got := b.file(t, filepath.Join(b.state, "found")); got != "always\n" {
t.Fatalf("found: %q", got)
}
if got := b.file(t, b.rotated); got != "--rotate\n" {
t.Fatalf("rotation: %q", got)
}
// Applied again (a restart with the same value): nothing written, nothing rotated, found kept.
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise, as set") {
t.Fatalf("again: %v %s", err, out)
}
if got := b.file(t, b.rotated); got != "--rotate\n" {
t.Fatalf("rotated again: %q", got)
}
if out, err := b.run(t, "restore"); err != nil || !strings.Contains(out, "back to always") {
t.Fatalf("restore: %v %s", err, out)
}
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "always\n" {
t.Fatalf("restored: %q", got)
}
// Nothing of the module is kept once it is stopped.
if _, err := os.Stat(filepath.Join(b.state, "found")); !os.IsNotExist(err) {
t.Fatalf("found kept after restore: %v", err)
}
}
func TestAReloadWithANewValueGoesStraightThereAndNeverRewritesFound(t *testing.T) {
dir := t.TempDir()
b := newBedIn(t, dir, "madvise", "[always] madvise never\n")
if out, err := b.run(t, "apply"); err != nil {
t.Fatalf("apply: %v %s", err, out)
}
// The setting changes to never: the mesh rewrites the script and reloads the unit, which applies again.
n := newBedIn(t, dir, "never", "")
out, err := n.run(t, "apply")
if err != nil || !strings.Contains(out, "madvise -> never") {
t.Fatalf("reload: %v %s", err, out)
}
if got := n.file(t, filepath.Join(n.state, "found")); got != "always\n" {
t.Fatalf("found rewritten by a reload: %q", got)
}
}
func TestNoRotationWhileTheMachineIsStarting(t *testing.T) {
for _, state := range []string{"initializing", "starting"} {
b := newBed(t, "madvise", "[always] madvise never\n")
b.booted(t, state)
out, err := b.run(t, "apply")
if err != nil || !strings.Contains(out, "still starting") {
t.Fatalf("%s: %v %s", state, err, out)
}
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "madvise\n" {
t.Fatalf("%s: not set: %q", state, got)
}
if got := b.file(t, b.rotated); got != "" {
t.Fatalf("%s: rotated at boot: %q", state, got)
}
}
}
func TestAlwaysIsSetWithoutRotating(t *testing.T) {
b := newBed(t, "always", "always [madvise] never\n")
if out, err := b.run(t, "apply"); err != nil || !strings.Contains(out, "madvise -> always") {
t.Fatalf("apply: %v %s", err, out)
}
if got := b.file(t, b.rotated); got != "" {
t.Fatalf("rotated for always: %q", got)
}
}
func TestAValueTheKernelDoesNotTakeIsRefusedBeforeAnythingIsWritten(t *testing.T) {
b := newBed(t, "sometimes", "[always] madvise never\n")
out, err := b.run(t, "apply")
if err == nil || !strings.Contains(out, "takes always, madvise or never") {
t.Fatalf("accepted: %v %s", err, out)
}
if got := b.file(t, filepath.Join(b.sys, "enabled")); got != "[always] madvise never\n" {
t.Fatalf("written anyway: %q", got)
}
}
func TestAWriteThatDoesNotTakeFailsTheUnit(t *testing.T) {
if os.Getuid() == 0 {
t.Skip("root writes a read-only file")
}
b := newBed(t, "madvise", "[always] madvise never\n")
if err := os.Chmod(filepath.Join(b.sys, "enabled"), 0o444); err != nil {
t.Fatal(err)
}
if out, err := b.run(t, "apply"); err == nil {
t.Fatalf("a refused write passed: %s", out)
}
}
func TestAKernelWithoutHugePagesIsSaid(t *testing.T) {
b := newBed(t, "madvise", "")
if out, err := b.run(t, "apply"); err == nil || !strings.Contains(out, "built without transparent huge pages") {
t.Fatalf("%v %s", err, out)
}
}
@@ -0,0 +1,79 @@
package main
import (
"encoding/json"
"os"
"testing"
)
type resource map[string]any
type manifestShape struct {
Module string `json:"module"`
Capabilities []string `json:"capabilities"`
Settings map[string]map[string]any `json:"settings"`
Tools []string `json:"tools"`
Resources []resource `json:"resources"`
Build struct {
Artifacts []map[string]any `json:"artifacts"`
} `json:"build"`
}
func manifest(t *testing.T) manifestShape {
t.Helper()
raw, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
var m manifestShape
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
return m
}
func (m manifestShape) resource(t *testing.T, id string) resource {
t.Helper()
for _, r := range m.Resources {
if r["id"] == id {
return r
}
}
t.Fatalf("no resource %s", id)
return nil
}
// TestToolsAreTheManifests holds the served tools and the manifest's list to one another, and the
// bundle to the shape the builder compiles and the runtime loads.
func TestToolsAreTheManifests(t *testing.T) {
m := manifest(t)
names := map[string]bool{}
for _, tool := range tools(&Machine{Root: t.TempDir()}) {
if names[tool.Name] {
t.Errorf("%s is served twice", tool.Name)
}
names[tool.Name] = true
}
for _, want := range m.Tools {
if !names[want] {
t.Errorf("the manifest lists %s and the bundle does not serve it", want)
}
delete(names, want)
}
if len(names) != 0 {
t.Errorf("served and not listed: %v", names)
}
var tools map[string]any
for _, a := range m.Build.Artifacts {
if a["name"] == "tools" {
tools = a
}
}
if tools == nil || tools["kind"] != "bundle" || tools["language"] != "go" || tools["system"] != "arch" ||
tools["from"] != "cmd/"+binaryName || tools["binary"] != binaryName {
t.Fatalf("the tools artifact: %v", tools)
}
if loads, _ := tools["loads"].([]any); len(loads) != 1 || loads[0] != binaryName {
t.Fatalf("loads: %v", tools["loads"])
}
}
+5
View File
@@ -0,0 +1,5 @@
module kernel
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.7
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.7 h1:C0sTQmtTiyYH7bnqZb7PusXnqA37gKuT7Nqjn9gG47w=
git.novox.be/novox/mesh-sdk/go v0.1.7/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+71
View File
@@ -0,0 +1,71 @@
{
"module": "kernel",
"version": "1",
"capabilities": [
"service-manager"
],
"settings": {
"transparent-huge-pages": {
"kind": "preference",
"default": "madvise",
"why": "madvise is the value the mesh chose for every node (novox/hq ADR 0381), not the always the distribution's kernel boots with; a machine that needs another value is given it as its own setting; madvise gives huge pages only to programs that ask for them, which stops a small write to a mapped file (the journal) dirtying and writing back a whole 2 MiB page on a machine whose disks are slow (novox/hq issue 746)"
}
},
"tools": [
"kernel_huge_pages"
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": "."
},
{
"id": "huge-pages",
"type": "file",
"path": "${dir:state}/huge-pages",
"mode": "0755",
"trusted": true,
"content": "#!/bin/sh\n# Transparent huge pages as the mesh sets them on this machine (module kernel, novox/hq issue 746).\n# Written by the mesh from this machine's setting transparent-huge-pages and run by\n# mesh-kernel-huge-pages.service: `apply` at boot and whenever the setting changes, `restore` when the\n# unit stops (the module unassigned, or the machine shutting down). Replaced on every push; change the\n# setting, never this file. A change of the setting reloads the unit, which runs `apply` again: the mode\n# moves straight to the new value and never passes back through the one the machine was found with.\n#\n# huge-pages apply|restore [<the kernel's transparent_hugepage directory> [<this module's state>]]\n#\n# The two directories are arguments only so the module's tests can run this against a stand-in tree.\nset -eu\n\nwant='${setting:transparent-huge-pages}'\nsys=/sys/kernel/mm/transparent_hugepage\nstate='${dir:state}'\n[ $# -ge 2 ] && sys=$2\n[ $# -ge 3 ] && state=$3\n\ncase \"$want\" in\nalways | madvise | never) ;;\n*)\n\techo \"the setting transparent-huge-pages is '$want'; the kernel takes always, madvise or never\" >&2\n\texit 2\n\t;;\nesac\n[ -f \"$sys/enabled\" ] || {\n\techo \"this kernel has no $sys/enabled: it was built without transparent huge pages, so there is nothing to set\" >&2\n\texit 1\n}\n\n# The mode in force is the word the kernel brackets: \"always [madvise] never\". A file holding one bare\n# word (the stand-in tree of the tests) is read as that word.\nmode() { sed -n -e 's/.*\\[\\([a-z]*\\)\\].*/\\1/p' -e t -e 's/^ *\\([a-z]*\\) *$/\\1/p' \"$sys/enabled\"; }\n\n[ $# -ge 1 ] || set -- usage\ncase \"$1\" in\napply)\n\tnow=$(mode)\n\t# What the machine had before the mesh first set it, kept once, so `restore` gives it back.\n\t[ -s \"$state/found\" ] || printf '%s\\n' \"$now\" >\"$state/found\"\n\tif [ \"$now\" = \"$want\" ]; then\n\t\techo \"transparent huge pages: $want, as set\"\n\t\texit 0\n\tfi\n\tprintf '%s\\n' \"$want\" >\"$sys/enabled\"\n\tgot=$(mode)\n\t[ \"$got\" = \"$want\" ] || {\n\t\techo \"wrote $want to $sys/enabled and the kernel says $got\" >&2\n\t\texit 1\n\t}\n\techo \"transparent huge pages: $now -> $want\"\n\t# A file folio already in the page cache keeps its size: the active journal files, mapped by journald\n\t# and by every reader, stay in 2 MiB folios until they are archived (issue 746). Rotating starts new\n\t# files that are mapped under the new mode. A failed rotation is said and does not fail the setting.\n\t# At boot nothing is in huge folios yet, and the persistent journal opens only after this unit: no rotation\n\t# while the service manager is still bringing the machine up.\n\tbooting=$(systemctl is-system-running 2>/dev/null || true)\n\tcase \"$booting\" in\n\tinitializing | starting) echo \"the machine is still starting: nothing to rotate yet\" ;;\n\t*)\n\t\tif [ \"$want\" != always ] && command -v journalctl >/dev/null 2>&1; then\n\t\t\tjournalctl --rotate || echo \"the journal could not be rotated: its active files keep their huge pages until they are archived\" >&2\n\t\tfi\n\t\t;;\n\tesac\n\t;;\nrestore)\n\tfound=$(cat \"$state/found\" 2>/dev/null || true)\n\tcase \"$found\" in\n\talways | madvise | never)\n\t\tprintf '%s\\n' \"$found\" >\"$sys/enabled\"\n\t\techo \"transparent huge pages: back to $found, as the machine had them\"\n\t\t;;\n\t*) echo \"transparent huge pages: nothing recorded as found, left at $(mode)\" ;;\n\tesac\n\t# Unassigned or shut down: nothing of the module is kept; the next start records what it finds again.\n\trm -f \"$state/found\"\n\t;;\n*)\n\techo \"usage: huge-pages apply|restore [<transparent_hugepage directory> [<state directory>]]\" >&2\n\texit 2\n\t;;\nesac\n"
},
{
"id": "unit",
"type": "file",
"path": "/etc/systemd/system/mesh-kernel-huge-pages.service",
"mode": "0644",
"content": "[Unit]\nDescription=Transparent huge pages as the mesh sets them on this machine\n# Module kernel, novox/hq issue 746. Starting this unit sets the mode from the setting\n# transparent-huge-pages; reloading it (the setting changed) sets the new mode; stopping it gives back the\n# mode the machine had before. It runs early in boot, before the services that fill the page cache.\nDefaultDependencies=no\nAfter=local-fs.target\nBefore=sysinit.target shutdown.target\nConflicts=shutdown.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=${dir:state}/huge-pages apply\nExecReload=${dir:state}/huge-pages apply\nExecStop=${dir:state}/huge-pages restore\n# It writes one file under /sys and its own state, and asks the journal daemon to rotate over its socket.\nProtectSystem=strict\nReadWritePaths=${dir:state}\nProtectHome=yes\nPrivateNetwork=yes\nNoNewPrivileges=yes\n\n[Install]\nWantedBy=sysinit.target\n"
},
{
"id": "apply",
"type": "service",
"unit": "mesh-kernel-huge-pages.service",
"state": "running",
"boot": "enabled",
"restart-on": [
"unit"
],
"reload-on": [
"huge-pages"
],
"health": {
"kind": "unit"
}
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/kernel-tools",
"binary": "kernel-tools",
"loads": [
"kernel-tools"
]
}
]
}
}
@@ -144,7 +144,9 @@ func (h *Holder) accounted(w Walk) bool {
return false return false
} }
// held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. // held is whether a live delivery follows the walk, or a live delivery it answers follows a newer one. A walk whose
// word only stops that lapsed give (lapsedOn, novox/hq issue 753) is not held: it stays under the orphans' timer, so
// past the grace without its word it is listed by `stalled`, saying why it was not given.
func (h *Holder) heldWalk(w Walk) bool { func (h *Holder) heldWalk(w Walk) bool {
for _, d := range h.deliveries { for _, d := range h.deliveries {
if d.State.Final() { if d.State.Final() {
@@ -292,12 +294,28 @@ func (h *Holder) orphanLines(now time.Time) []StalledLine {
continue continue
} }
w := s.walk w := s.walk
if lapsed, behind := h.lapsedOf(w); len(lapsed) > 0 || len(behind) > 0 {
// Owed its word by stops that lapsed, and not given it past the grace (review of issue 753): said, why.
says := fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, which the stopped deliveries whose "+
"merge it answers owe it (novox/hq issue 753), and it has not started", w.ID, w.Repository, shortOf(w.Commit, 8))
if why := h.lapsedSaid(w); why != "" {
says += ": " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: its word is given at its turn — the operator's past the bound", Says: says})
continue
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID, out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: w.ID,
For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(), For: now.Sub(first).Round(time.Second).String(), Bound: matchGrace.String(),
H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's", H2: "none: no delivery holds the walk, so nothing gives it its word — the operator's",
Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+ Says: fmt.Sprintf("the walk %s of %s at %s waits for this owner's word, and no delivery that has not ended "+
"follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+ "follows it or is answered by it: `mesh-delivery.show` for the deliveries of %s, and `plans go %s --why …` "+
"starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)}) "starts it by hand", w.ID, w.Repository, shortOf(w.Commit, 8), w.Repository, w.ID)})
out[len(out)-1].Says += h.endedSaid(w)
if hold := h.coverHeld(id); hold != "" {
out[len(out)-1].Says += "; it is held back meanwhile: " + hold
}
} }
return out return out
} }
@@ -344,6 +362,7 @@ func (h *Holder) forgetWalks(now time.Time) {
} }
case !w.Open() && now.Sub(s.at) > keptWalkFor: case !w.Open() && now.Sub(s.at) > keptWalkFor:
delete(h.walks, id) delete(h.walks, id)
delete(h.deliverErr, id)
} }
} }
} }
@@ -356,6 +375,168 @@ type turnWant struct {
whys []string whys []string
} }
// endedSaid names the ended deliveries a waiting walk answers or that follow it, with their states, so a walk no live
// delivery holds says what it carries instead of only that nothing holds it (novox/hq issue 753); empty for none.
func (h *Holder) endedSaid(w Walk) string {
var ended []string
for _, d := range h.deliveries {
if !d.State.Final() {
continue
}
_, answers := walkAnswers(w, d)
if answers || d.Walk != nil && d.Walk.ID == w.ID {
ended = append(ended, fmt.Sprintf("%s (%s)", d.ID, d.State))
}
}
if len(ended) == 0 {
return ""
}
sort.Strings(ended)
return "; every delivery it carries ended: " + strings.Join(ended, ", ")
}
// lapsedOn are the deliveries, by id, whose word an open walk waiting for this owner's word takes from a stop that
// lapsed (novox/hq issue 753): each was stopped by its group for a member delivered since, so its own change waits for
// a later walk of its repository that contains its merge (issue 747), and this walk answers that merge — it names it,
// or is made at it — and has not left it out. Such a walk is the one that delivery waits for: it gives it its word, so
// a walk whose every other delivery ended is never left without one. A walk at the merge of a delivery stopped for
// itself, or by a person, gets nothing from it. Under the lock.
//
// In its group's order (review of issue 753): a member gives its word only once every member ordered before it reached
// the machines, or is itself a stop that lapsed whose merge this same walk answers — the walk then delivers both, in
// the group's order among its tiers, as members sharing one walk do. A member behind one still undelivered gives
// nothing; lapsedBehind names that one.
func (h *Holder) lapsedOn(w Walk) []string {
out, _ := h.lapsedOf(w)
return out
}
// lapsedOf is lapsedOn, and, by delivery, the member before it in its group that holds back the word of a stop that
// lapsed whose merge the walk answers. Under the lock.
func (h *Holder) lapsedOf(w Walk) ([]string, map[string]string) {
if w.Release != nil || w.IsBatch() || !w.Open() || !w.Seen().Waits {
return nil, nil
}
var out []string
behind := map[string]string{}
for _, d := range h.deliveries {
if !h.lapsedAnswered(w, d) {
continue
}
if before := h.lapsedBehind(w, d); before != "" {
behind[d.ID] = before
continue
}
out = append(out, d.ID)
}
sort.Strings(out)
return out, behind
}
// lapsedAnswered is whether a delivery's stop lapsed and the walk answers its merge, not left out.
func (h *Holder) lapsedAnswered(w Walk, d *Delivery) bool {
if d.MergedAs == "" || !h.stopLapsed(d) {
return false
}
if _, ok := walkAnswers(w, d); !ok {
return false
}
_, left := w.leftOut(d.Repository, d.MergedAs)
return !left
}
// lapsedBehind is the member ordered before a delivery in its group that has not reached the machines and that this
// walk does not deliver with it, with its state; empty for none.
func (h *Holder) lapsedBehind(w Walk, d *Delivery) string {
g := h.groups[d.Group]
if d.Group == "" || g == nil {
return ""
}
members := h.membersOf(g)
byID := map[string]*Delivery{}
for _, m := range members {
byID[m.ID] = m
}
for _, m := range h.inOrder(g, members) {
if m.ID == d.ID {
return ""
}
o := byID[m.ID]
if o == nil || memberReached(o) || h.lapsedAnswered(w, o) {
continue
}
return fmt.Sprintf("%s (%s)", o.ID, o.State)
}
return ""
}
// lapsedSaid is why a walk whose word stops that lapsed give still waits: the members of their groups not yet
// delivered before them, what the last tick held it for, or the controller's refusal of the word. Under the lock.
func (h *Holder) lapsedSaid(w Walk) string {
var parts []string
_, behind := h.lapsedOf(w)
ids := make([]string, 0, len(behind))
for id := range behind {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
parts = append(parts, fmt.Sprintf("%s waits for %s, ordered before it in its group", id, behind[id]))
}
if why := h.waitsWhy[w.ID]; why != "" {
parts = append(parts, "not let go yet: "+why)
}
if err := h.deliverErr[w.ID]; err != "" {
parts = append(parts, "the controller refused its word: "+err)
}
lapsed, _ := h.lapsedOf(w)
for _, id := range lapsed {
if d := h.deliveries[id]; d != nil && strings.Contains(d.LetGoWhy, lapsedWord(w.ID)) {
parts = append(parts, fmt.Sprintf("its word was given for %s at %s, and the controller has not started it",
id, d.LetGo.Local().Format("15:04:05")))
}
}
return strings.Join(parts, "; ")
}
// lapsedOwed is whether a walk's word is owed by stops that lapsed: given now, or once a member before one is delivered.
func (h *Holder) lapsedOwed(w Walk) bool {
lapsed, behind := h.lapsedOf(w)
return len(lapsed) > 0 || len(behind) > 0
}
// lapsedTurns registers, for every open walk waiting for this owner's word, the turn of each delivery whose stop
// lapsed and whose merge it answers (lapsedOn, novox/hq issue 753). The word is then given as for any walk: once,
// when every live delivery it carries is at its turn too and it carries no stopped change still held back (words).
// Given once per walk: the walk's id is in the delivery's LetGoWhy once its word was given. Under the lock.
func (h *Holder) lapsedTurns() {
for id, s := range h.walks {
for _, did := range h.lapsedOn(s.walk) {
d := h.deliveries[did]
if strings.Contains(d.LetGoWhy, lapsedWord(id)) {
continue
}
if h.turning == nil {
h.turning = map[string]*turnWant{}
}
want := h.turning[id]
if want == nil {
want = &turnWant{}
h.turning[id] = want
}
if !containsString(want.ids, d.ID) {
want.ids = append(want.ids, d.ID)
want.whys = append(want.whys, fmt.Sprintf("%s: %s its merge %s, stopped by its group for %s, whose change "+
"was delivered since; it waits for a later walk of %s that contains its merge, and this is that "+
"walk (novox/hq issue 753)", d.ID, lapsedWord(id), shortOf(d.MergedAs, 8), d.stoppedFor(), d.Repository))
}
}
}
}
// lapsedWord is how the word a lapsed stop gives a walk names that walk.
func lapsedWord(walk string) string { return "the walk " + walk + " answers" }
// words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per // words are the asks of the word of every walk whose turn came for every live delivery it carries: one ask per
// walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5). // walk, however many deliveries it carries, and none while one of them is not at its turn (ADR 0276 decision 5).
// Under the lock. // Under the lock.
@@ -388,6 +569,9 @@ func (h *Holder) words() []func() {
if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 { if stopped, _ := h.carriesStopped(walk); len(stopped) > 0 {
h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " + h.waitsWhy[walk] = "it carries " + strings.Join(stopped, ", ") + ", whose delivery was stopped: it is " +
"asked to leave that merge out first" "asked to leave that merge out first"
if hold := h.coverHeld(walk); hold != "" {
h.waitsWhy[walk] = hold
}
h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk, h.Logf("[mesh-delivery] the walk %s is not let go: it carries %s, whose delivery was stopped", walk,
strings.Join(stopped, ", ")) strings.Join(stopped, ", "))
continue continue
@@ -403,9 +587,14 @@ func (h *Holder) words() []func() {
defer h.mu.Unlock() defer h.mu.Unlock()
delete(h.asking, walk) delete(h.asking, walk)
if err != nil { if err != nil {
if h.deliverErr == nil {
h.deliverErr = map[string]string{}
}
h.deliverErr[walk] = err.Error() // said by `stalled` past the grace (review of novox/hq issue 753)
h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err) h.Logf("[mesh-delivery] the walk %s could not be let go yet: %v", walk, err)
return return
} }
delete(h.deliverErr, walk)
for _, id := range ids { for _, id := range ids {
d := h.deliveries[id] d := h.deliveries[id]
if d == nil { if d == nil {
@@ -479,9 +668,28 @@ func (h *Holder) WalksWaiting() []StalledLine {
switch { switch {
case len(held) > 0: case len(held) > 0:
out = append(out, held...) out = append(out, held...)
case h.coverHeld(id) != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: coverAskBound.String(), H2: "none: it is asked to leave the merge out past the bound",
Says: "held back: " + h.coverHeld(id)})
case h.waitsWhy[id] != "": case h.waitsWhy[id] != "":
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited, out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]}) Bound: "0s", H2: "none: it goes at its turn", Says: "not let go yet: " + h.waitsWhy[id]})
case !live && h.lapsedOwed(w):
// Its word comes from the stopped deliveries whose merge it answers (novox/hq issue 753): said, never
// unheld, while the controller has not taken it, with what holds it.
lapsed, _ := h.lapsedOf(w)
says := "its word is owed by the stopped deliveries whose merge it answers, each stopped for a member " +
"delivered since (novox/hq issue 753)"
if len(lapsed) > 0 {
says = "its word is given for " + strings.Join(lapsed, ", ") + ", stopped for a member delivered since, " +
"whose merge it answers (novox/hq issue 753)"
}
if why := h.lapsedSaid(w); why != "" {
says += "; " + why
}
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Published, Walk: id, For: waited,
Bound: "0s", H2: "none: it goes at its turn", Says: says})
case !live: case !live:
says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " + says := "no delivery that has not ended follows it or is answered by it, so nothing gives it its word: " +
"`mesh-delivery.show` for the deliveries of " + w.Repository "`mesh-delivery.show` for the deliveries of " + w.Repository
@@ -492,7 +700,7 @@ func (h *Holder) WalksWaiting() []StalledLine {
"), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word" "), yet the walk moves " + strings.Join(walkModules(w), ", ") + " — nothing gives it its word"
} }
out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited, out = append(out, StalledLine{ID: IDOf(w.Repository, w.Commit), State: Unheld, Walk: id, For: waited,
Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says}) Bound: "0s", H2: "none: no delivery holds the walk — the operator's", Says: says + h.endedSaid(w)})
} }
} }
sort.Slice(out, func(i, j int) bool { sort.Slice(out, func(i, j int) bool {
@@ -150,6 +150,9 @@ func statusOf(d *Delivery) (string, string) {
case Delivered: case Delivered:
return "success", "delivered" return "success", "delivered"
case Failed: case Failed:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its change reached the machines (novox/hq issue 747)
}
return "failure", clip("failed: "+why, 140) return "failure", clip("failed: "+why, 140)
case Superseded: case Superseded:
return "warning", clip("superseded: "+why, 140) return "warning", clip("superseded: "+why, 140)
@@ -178,7 +181,7 @@ func ViewBody(d *Delivery, g *Group) string {
var b strings.Builder var b strings.Builder
fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339)) fmt.Fprintf(&b, "%s\n**Delivery** `%s` — **%s** since %s\n\n", viewMarker, d.ID, d.State, d.Since.UTC().Format(time.RFC3339))
if d.coveredStop() { if d.coveredStop() {
fmt.Fprintf(&b, "**Covered** after its stop: a later merge that contains it was delivered: %s\n\n", d.CoveredBy) fmt.Fprintf(&b, "**Covered**: %s\n\n", d.coveredStopSaid())
} }
if d.HeldWhy != "" && d.State == Held { if d.HeldWhy != "" && d.State == Held {
fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy) fmt.Fprintf(&b, "Held for a person: %s — `mesh-delivery.release` with why lets it go on.\n\n", d.HeldWhy)
@@ -106,6 +106,8 @@ type fakeController struct {
moves map[string][]string moves map[string][]string
plans int plans int
down bool down bool
// refuseDeliver refuses every word, as a controller that will not take it does (review of novox/hq issue 753).
refuseDeliver bool
// onCheck runs as a check is asked, before it is answered: what the forge held at that moment. // onCheck runs as a check is asked, before it is answered: what the forge held at that moment.
onCheck func() onCheck func()
// planned is what Plan answers instead of "builds app", when set. // planned is what Plan answers instead of "builds app", when set.
@@ -182,6 +184,9 @@ func (c *fakeController) Deliver(walk, why string) error {
if c.down { if c.down {
return errors.New("down") return errors.New("down")
} }
if c.refuseDeliver {
return errors.New("the controller refuses the word")
}
w, known := c.walks[walk] w, known := c.walks[walk]
if !known || w.Delivery == nil || w.Delivery.Go != nil { if !known || w.Delivery == nil || w.Delivery.Go != nil {
return fmt.Errorf("%s waits for nobody", walk) return fmt.Errorf("%s waits for nobody", walk)
+307 -26
View File
@@ -60,6 +60,8 @@ type Holder struct {
waitsWhy map[string]string waitsWhy map[string]string
// asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile. // asking are the walks whose word is being asked right now, outside the lock: not asked twice meanwhile.
asking map[string]bool asking map[string]bool
// deliverErr is, by walk, the controller's last refusal of its word, until it is given (review of issue 753).
deliverErr map[string]string
// leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299). // leaving are, by walk, the merges a stop gathered to leave out of it, asked once (novox/hq ADR 0299).
leaving map[string][]string leaving map[string][]string
// leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped). // leaveAsked is when a walk was last asked to leave out the stopped merges it carries (leaveStopped).
@@ -67,6 +69,13 @@ type Holder struct {
// leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and // leaveRefused is, by walk, what the controller answered when it refused both to leave a stopped merge out and
// to stop the walk whole. // to stop the walk whole.
leaveRefused map[string]string leaveRefused map[string]string
// coverAsked are the deliveries that failed or were stopped whose cover by a later finished walk waits for the
// forge's word on containment, worked out each pass (novox/hq issue 747): a walk carrying a merge stopped for one of
// them is held back meanwhile, never asked to leave it out before the answer.
coverAsked map[string]bool
// coverAskedSince is, by delivery, when the forge was first asked about its cover and has not answered: past
// coverAskBound the walk is no longer held back for it, and is asked to leave the merge out (review of issue 747).
coverAskedSince map[string]time.Time
// tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407).
tried owedTries tried owedTries
// marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go). // marks are the open pull requests read with a ready mark, by owner/repo#number (novox/hq issue 722, ready.go).
@@ -303,19 +312,25 @@ func (h *Holder) PullUpdated(p PullEvent) {
defer h.mu.Unlock() defer h.mu.Unlock()
id := IDOf(p.repository(), p.HeadSHA) id := IDOf(p.repository(), p.HeadSHA)
now := h.Now() now := h.Now()
startedOver := ""
if d := h.deliveries[id]; d != nil { if d := h.deliveries[id]; d != nil {
// Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its if startedOver = startedOverFor(d, p.Number); startedOver == "" {
// check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound // Announced again: nothing new — but for a delivery whose verdict is out of date, the announcement is its
// runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it. // check asked again against the trunk's head (the merge tool's ask, novox/hq ADR 0323 §5), and its bound
if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA { // runs from now (§7). The base head it was announced with is kept, for a check this owner asks after it.
d.BaseSHA = p.BaseSHA if p.BaseSHA != "" && d.BaseSHA != p.BaseSHA {
h.keep(d) d.BaseSHA = p.BaseSHA
h.keep(d)
}
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil {
d.AskedAgain = &now
h.keep(d)
}
return
} }
if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil { // A delivery stopped only because its pull request closed unmerged does not end its commit's delivery for
d.AskedAgain = &now // another pull request with the same head (novox/hq issue 783): started over below, exactly as for a first
h.keep(d) // announcement, under the same id — never a transition out of Stopped.
}
return
} }
replaced := "" replaced := ""
for _, old := range h.deliveries { for _, old := range h.deliveries {
@@ -329,11 +344,7 @@ func (h *Holder) PullUpdated(p PullEvent) {
} }
} }
} }
d := &Delivery{ID: id, Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title, d, t, err := h.freshAnnouncement(p, now, startedOver)
Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)})
if err != nil { if err != nil {
h.refuse(err) h.refuse(err)
return return
@@ -351,6 +362,34 @@ func (h *Holder) PullUpdated(p PullEvent) {
} }
} }
// startedOverFor is why a delivery already kept at a head's id is started over for a pull request's number rather
// than taken as announced again: only when it was stopped solely by an earlier, closed duplicate pull request with
// the same head — still Stopped, ClosedUnmerged, never merged, and under another number (novox/hq issue 783).
// Empty for the ordinary 'announced again' path, or for any other stop (a person's, a group's, its own close).
func startedOverFor(d *Delivery, number int) string {
if d != nil && d.State == Stopped && d.ClosedUnmerged && d.MergedAs == "" && d.Number != number {
return fmt.Sprintf("#%d with the same head closed unmerged", d.Number)
}
return ""
}
// freshAnnouncement makes a delivery for p's head exactly as a first announcement would: its own id, proposed.
// startedOver, when not empty, is folded into the transition's Why, naming the pull request this replaces
// (novox/hq issue 783); the delivery it replaces is swapped out by the caller keeping the new one under the same
// id (keep), never by a transition out of its final state.
func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver string) (*Delivery, Transition, error) {
d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number,
Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths,
PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid,
After: AfterIn(p.Body), Created: now, BaseSHA: p.BaseSHA}
why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number)
if startedOver != "" {
why = fmt.Sprintf("%s; %s", why, startedOver)
}
t, err := Apply(d, EvAnnounced, false, Facts{Now: now, Why: why})
return d, t, err
}
// takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded, // takePlaceOf puts a member's new head — a rebase, a push — in its group in the place of the head it superseded,
// when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The // when the group is closed: a member of it already reached the trunk, and a closed group takes no new member. The
// new head has its own check, and the group's composed check runs again for the new heads rather than holding the // new head has its own check, and the group's composed check runs again for the new heads rather than holding the
@@ -482,7 +521,29 @@ func (h *Holder) PullMerged(p PullEvent) {
} }
now := h.Now() now := h.Now()
var ts []Transition var ts []Transition
if d == nil { if over := startedOverFor(d, p.Number); over != "" {
// A delivery stopped only by an earlier, closed duplicate pull request with the same head is not lost when
// its merge is heard without the open announcement either: it is started over for the merging pull request
// the same way an announcement would, before the merge is taken as usual (novox/hq issue 783). Heard this
// way — its own merge, with no open announcement of this pull request — it is made exactly as a first-heard
// merge in every field but its id: FromForge, and its check the forge's word on the head, so a long-past
// merge replayed from the bus's history still falls under the history rule (issue 313) and is not held for
// a person (review round 1).
fresh, t, err := h.freshAnnouncement(p, now, over)
if err != nil {
h.refuse(err)
return
}
fresh.FromForge = true
if s := p.HeadChecks["mesh/merge-gate"]; s != "" {
fresh.Check = &Verdict{At: now, Gate: verdictOfState(s), Summary: "the head's mesh/merge-gate, as the forge holds it",
Repo: verdictOfState(p.HeadChecks["mesh/repo-check"])}
}
d, ts = fresh, []Transition{t}
if d.Check != nil {
ts = append(ts, settle(d, Facts{Now: now})...)
}
} else if d == nil {
head := p.HeadSHA head := p.HeadSHA
if head == "" { if head == "" {
head = p.MergeCommit head = p.MergeCommit
@@ -1112,6 +1173,48 @@ func (h *Holder) plan() []func() {
// A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what // A published delivery whose walk ended done before its word is read the same way (novox/hq issue 474): what
// that walk did not build, a later delivered merge may have; and when the cover test says none did, past the // that walk did not build, a later delivered merge may have; and when the cover test says none did, past the
// grace, it is stopped saying so — never left waiting for a start that cannot come. // grace, it is stopped saying so — never left waiting for a start that cannot come.
// A delivery that failed or was stopped is read for a later finished walk of its repository that delivered its
// change after all: a walk that failed in an outage and was resumed, or the walk of a later merge that contains it
// (novox/hq issue 747). Read each tick, from what the walks and the kept deliveries say, so one stuck before this
// was built clears by itself after a restart.
h.coverAsked = map[string]bool{}
if h.coverAskedSince == nil {
h.coverAskedSince = map[string]time.Time{}
}
pending := map[string]bool{}
for _, d := range h.deliveries {
if d.State != Failed && d.State != Stopped || d.coveredStop() {
continue
}
by, ask := h.walkCover(d)
if len(ask) > 0 && by == "" {
pending[d.ID] = true
since, asked := h.coverAskedSince[d.ID]
if !asked {
since = now
h.coverAskedSince[d.ID] = now
}
// Held back only within the bound: a forge that keeps erroring never holds a walk for good.
h.coverAsked[d.ID] = now.Sub(since) < coverAskBound
}
if by != "" {
d.DeliveredBy = by
h.tellCovered(d, now)
continue
}
for _, in := range ask {
if a := h.askContains(d, in, now); a != nil {
asks = append(asks, a)
}
}
}
for id := range h.coverAskedSince {
if !pending[id] {
delete(h.coverAskedSince, id)
}
}
for _, d := range h.deliveries { for _, d := range h.deliveries {
ended := d.endedUnstarted() ended := d.endedUnstarted()
if d.coveredStop() && !d.CoverTold { if d.coveredStop() && !d.CoverTold {
@@ -1195,6 +1298,8 @@ func (h *Holder) plan() []func() {
} }
h.turn(d, "its turn: a delivery in no group") h.turn(d, "its turn: a delivery in no group")
} }
// A walk that answers the merge of a delivery whose stop lapsed: that delivery's turn (novox/hq issue 753).
h.lapsedTurns()
// The word, once per walk, when every live delivery it carries is at its turn. // The word, once per walk, when every live delivery it carries is at its turn.
asks = append(asks, h.words()...) asks = append(asks, h.words()...)
@@ -1273,7 +1378,7 @@ func (h *Holder) laterOnItsWay(d *Delivery) string {
// stays final, and the bus's transition event is not said again. // stays final, and the bus's transition event is not said again.
func (h *Holder) tellCovered(d *Delivery, now time.Time) { func (h *Holder) tellCovered(d *Delivery, now time.Time) {
d.CoverTold = true d.CoverTold = true
line := oneLine(fmt.Sprintf("%s delivery %s: stopped, %s", now.UTC().Format(time.RFC3339), d.ID, d.coveredStopSaid())) line := oneLine(fmt.Sprintf("%s delivery %s: %s, %s", now.UTC().Format(time.RFC3339), d.ID, d.State, d.coveredStopSaid()))
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now}) d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.Commit, Line: line, Since: now})
if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) { if d.MergedAs != "" && !sameCommit(d.MergedAs, d.Commit) {
d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now}) d.Owed = append(d.Owed, Effect{Kind: EffectNote, Commit: d.MergedAs, Line: line, Since: now})
@@ -1283,7 +1388,7 @@ func (h *Holder) tellCovered(d *Delivery, now time.Time) {
state, desc := statusOf(d) state, desc := statusOf(d)
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now}) d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
} }
h.Logf("[mesh-delivery] %s: stopped, %s", d.ID, d.coveredStopSaid()) h.Logf("[mesh-delivery] %s: %s, %s", d.ID, d.State, d.coveredStopSaid())
h.keep(d) h.keep(d)
} }
@@ -1294,6 +1399,104 @@ func strandedUncovered(d *Delivery) bool {
return d.State == Stopped && d.Stranded != "" && d.CoveredBy == "" return d.State == Stopped && d.Stranded != "" && d.CoveredBy == ""
} }
// walkCover says which later finished walk delivered the change of a delivery that failed or was stopped, or
// nothing, and which walk commits the forge must still be asked about (novox/hq issue 747). A walk covers it when it
// is of its repository and trunk, done, made after its merge, its commit containing its merge — the merge itself, or
// a later one the forge says contains it — and it built every module the delivery's plan moves, each it did not
// build being one the delivery's own walk said unchanged (issue 611). This is the containment "one walk answers every
// merge it contains" uses (issue 362, ADR 0276), read after the fact: a failed walk resumed to done answers its own
// merge, and a later merge's walk answers every merge it contains. A walk that only said a module unchanged for its
// own merges says nothing of this one's, so it covers no module that way. Walks are read from the controller's records
// this owner holds and from every delivery's kept walk, so a walk long ended still counts after a restart.
func (h *Holder) walkCover(d *Delivery) (string, []string) {
if d.State != Failed && d.State != Stopped || d.MergedAs == "" || d.Plan == nil || len(d.Plan.Moved) == 0 {
return "", nil
}
var ask []string
for _, c := range h.doneWalksOf(d.Repository) {
w := c.walk
if !sameBranch(w.Branch, d.Base) || !w.Created.IsZero() && w.Created.Before(d.MergedAt.Add(-time.Minute)) {
continue
}
var missing []string
for _, m := range d.Plan.Moved {
unchanged := false
if d.Walk != nil {
_, unchanged = d.Walk.Unchanged[m]
}
if !containsString(w.Moved, m) && !unchanged {
missing = append(missing, m)
}
}
if len(missing) > 0 {
continue
}
if !sameCommit(c.commit, d.MergedAs) {
yes, known := d.ContainedIn[c.commit]
if !known && !containsString(ask, c.commit) {
ask = append(ask, c.commit)
}
if !yes {
continue
}
}
return fmt.Sprintf("%s (%s at %s into %s, done, contains its merge %s)", w.ID, d.Repository, shortOf(c.commit, 8),
orNone(d.Base), shortOf(d.MergedAs, 8)), nil
}
return "", ask
}
// doneWalk is a finished walk and the commit of its repository it carried.
type doneWalk struct {
walk *WalkSeen
commit string
}
// doneWalksOf are the finished walks of a repository this owner knows, oldest first: the controller's records it
// holds, and each delivery's kept walk with the commit of its repository that walk carried for it.
func (h *Holder) doneWalksOf(repository string) []doneWalk {
byID := map[string]doneWalk{}
for id, s := range h.walks {
w := s.walk
if w.Release != nil || w.IsBatch() || w.State != walkDone || !strings.EqualFold(w.Repository, repository) {
continue
}
byID[id] = doneWalk{walk: w.Seen(), commit: w.Commit}
}
for _, o := range h.deliveries {
if o.Walk == nil || o.Walk.State != walkDone || !strings.EqualFold(o.Repository, repository) {
continue
}
if _, known := byID[o.Walk.ID]; known {
continue
}
commit := o.AnsweredBy
if commit == "" && o.MergedAs != "" && sameCommit(o.Walk.Commit, o.MergedAs) {
commit = o.MergedAs
}
if commit == "" {
continue
}
byID[o.Walk.ID] = doneWalk{walk: o.Walk, commit: commit}
}
out := make([]doneWalk, 0, len(byID))
for _, c := range byID {
out = append(out, c)
}
sort.Slice(out, func(i, j int) bool {
if !out[i].walk.Created.Equal(out[j].walk.Created) {
return out[i].walk.Created.Before(out[j].walk.Created)
}
return out[i].walk.ID < out[j].walk.ID
})
return out
}
// stopLapsed is whether a member its group stopped for another is stopped for nothing any more (see stopLapsed).
func (h *Holder) stopLapsed(d *Delivery) bool {
return stopLapsed(d, func(id string) *Delivery { return h.deliveries[id] })
}
// unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read // unwalkedPast is whether a merged delivery has gone without a walk past the bound one opens within, read
// again after it: what holds it for a person, unless something already ends it. // again after it: what holds it for a person, unless something already ends it.
func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool { func (h *Holder) unwalkedPast(d *Delivery, now time.Time) bool {
@@ -1736,7 +1939,8 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
// A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a // A superseded member a later delivered walk carried counts as reached, as in GroupState: a group counts a
// covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq // covered member as delivered, so the member after it gets its turn (ADR 0270 decision 2, novox/hq
// issues 442 and 445). // issues 442 and 445).
if d.State.Reached() || d.State == Superseded && d.CoveredBy != "" { // A stop or failure a later walk delivered since counts as reached too (novox/hq issues 480 and 747).
if memberReached(d) {
continue continue
} }
if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) { if sharing != "" && (d.Walk == nil || d.Walk.ID != sharing) {
@@ -1761,7 +1965,7 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
} }
if w.why != "" { if w.why != "" {
for _, later := range ordered[i+1:] { for _, later := range ordered[i+1:] {
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID, if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s superseded, and %s", g.ID,
d.ID, w.why)); a != nil { d.ID, w.why)); a != nil {
asks = append(asks, a) asks = append(asks, a)
} }
@@ -1788,9 +1992,13 @@ func (h *Holder) turnsOf(g *Group, members []*Delivery, waits map[string]carryWa
continue continue
} }
return asks return asks
case h.stopLapsed(d):
// Stopped for a member whose change reached the machines since: it stops nothing more. Its own change is
// delivered by a later walk of its repository that contains it, which then covers it (novox/hq issue 747).
continue
case d.State == Failed || d.State == Stopped: case d.State == Failed || d.State == Stopped:
for _, later := range ordered[i+1:] { for _, later := range ordered[i+1:] {
if a := h.stopInGroup(byID[later.ID], fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil { if a := h.stopInGroup(byID[later.ID], d.ID, fmt.Sprintf("its group %s stopped: %s %s", g.ID, d.ID, d.State)); a != nil {
asks = append(asks, a) asks = append(asks, a)
} }
} }
@@ -2028,7 +2236,7 @@ func mayCarry(c, d *Delivery, module string) bool {
// controller ends the walk and walks the other merges again without it, and without each later merge of its // controller ends the walk and walks the other merges again without it, and without each later merge of its
// repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked // repository, which contains it. Every member a pass stops on one walk is gathered first, and the walk is asked
// once, by the first member's ask, after the pass: a member is never walked again because its stop came second. // once, by the first member's ask, after the pass: a member is never walked again because its stop came second.
func (h *Holder) stopInGroup(d *Delivery, why string) func() { func (h *Holder) stopInGroup(d *Delivery, cause, why string) func() {
if d == nil || d.State.Final() { if d == nil || d.State.Final() {
return nil return nil
} }
@@ -2037,6 +2245,7 @@ func (h *Holder) stopInGroup(d *Delivery, why string) func() {
h.refuse(err) h.refuse(err)
return nil return nil
} }
d.StoppedFor = cause // the member it was stopped for: once that one is delivered, the stop lapses (novox/hq issue 747)
h.moved(d, []Transition{t}) h.moved(d, []Transition{t})
// Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review // Its open walk; or, its walk stopped for another merge, the batch or walk that walks its merge again (review
// of #221): a merge walked again and then stopped by its group is left out there too. // of #221): a merge walked again and then stopped by its group is left out there too.
@@ -2118,6 +2327,12 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" { if last := d.Transitions[len(d.Transitions)-1]; last.Event != EvStop || last.By == "" {
continue continue
} }
// A stop whose change a later walk delivered since, or one its group made for a member delivered since,
// holds nothing back: the group stops a walk only for a member whose change is still undelivered (novox/hq
// issue 747).
if d.coveredStop() || h.stopLapsed(d) {
continue
}
// By the merges it names alone: its own commit may be another's merge it no longer answers. // By the merges it names alone: its own commit may be another's merge it no longer answers.
if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool { if !slices.ContainsFunc(s.walk.Delivery.Merges, func(m WalkMerge) bool {
return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs) return strings.EqualFold(m.Repository, d.Repository) && sameCommit(m.Commit, d.MergedAs)
@@ -2135,6 +2350,51 @@ func (h *Holder) carriesStopped(walk string) ([]string, string) {
return out, why return out, why
} }
// coverAwaited is whether a stopped merge a walk carries, given as `<repository>@<commit>`, waits for the forge's
// word on whether a later finished walk delivered it or the member its group stopped it for (novox/hq issue 747).
// It says since when the forge was first asked, the earliest of them. Held only within coverAskBound of the first
// ask: past it the forge's silence or error lets the walk be asked to leave the merge out. Under the lock.
func (h *Holder) coverAwaited(merges []string) (time.Time, bool) {
var since time.Time
awaited := false
for _, d := range h.deliveries {
if d.State != Stopped || d.MergedAs == "" || !containsString(merges, d.Repository+"@"+d.MergedAs) {
continue
}
for _, id := range []string{d.ID, d.stoppedFor()} {
if id == "" || !h.coverAsked[id] {
continue
}
if at := h.coverAskedSince[id]; !awaited || at.Before(since) {
since = at
}
awaited = true
}
}
return since, awaited
}
// coverHeld says why a walk is held back while the forge is asked whether a later finished walk delivered a stopped
// merge it carries (novox/hq issue 747), or nothing when it is not. Said by `waits`, `stalled` and `stalled walks`, so
// a forge that cannot answer is loud. Under the lock.
func (h *Holder) coverHeld(walk string) string {
without, _ := h.carriesStopped(walk)
if len(without) == 0 {
return ""
}
since, awaited := h.coverAwaited(without)
if !awaited {
return ""
}
return fmt.Sprintf("it carries %s, whose delivery was stopped, and the forge is asked since %s whether a later "+
"finished walk delivered it; it is asked to leave the merge out if no answer comes by %s (novox/hq issue 747)",
strings.Join(without, ", "), since.Local().Format("15:04:05"), since.Add(coverAskBound).Local().Format("15:04:05"))
}
// coverAskBound is how long a walk carrying a stopped merge is held back while the forge is asked whether a later
// finished walk delivered it (novox/hq issue 747); past it, the walk is asked to leave the merge out.
const coverAskBound = 10 * time.Minute
// lastWhyOf is the reason of a delivery's last transition. // lastWhyOf is the reason of a delivery's last transition.
func lastWhyOf(d *Delivery) string { func lastWhyOf(d *Delivery) string {
if len(d.Transitions) == 0 { if len(d.Transitions) == 0 {
@@ -2170,6 +2430,9 @@ func (h *Holder) leaveStopped() []func() {
if without, _ := h.carriesStopped(id); len(without) > 0 { if without, _ := h.carriesStopped(id); len(without) > 0 {
said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+ said := fmt.Sprintf("its walk %s is not let go: it carries %s, whose delivery was stopped, and is asked to "+
"leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", ")) "leave it out (novox/hq ADR 0299)", id, strings.Join(without, ", "))
if hold := h.coverHeld(id); hold != "" {
said = "its walk " + id + " is held back: " + hold
}
if r := h.leaveRefused[id]; r != "" { if r := h.leaveRefused[id]; r != "" {
said += "; the controller refused: " + r said += "; the controller refused: " + r
} }
@@ -2193,6 +2456,9 @@ func (h *Holder) leaveStopped() []func() {
delete(h.leaveRefused, id) delete(h.leaveRefused, id)
continue continue
} }
if _, awaited := h.coverAwaited(without); awaited {
continue // held back, not stopped, until the forge says whether a later walk delivered it (issue 747)
}
// Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should // Once per record of it: the record that answers an ask is newer. Asked again after five minutes, should
// the ask have gone nowhere. // the ask have gone nowhere.
if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute { if a, asked := h.leaveAsked[id]; asked && seen.walk.Revision <= a.revision && now.Sub(a.at) < 5*time.Minute {
@@ -2314,18 +2580,28 @@ func againWords(a StopAnswer) string {
func GroupState(g *Group, members []*Delivery) (string, string) { func GroupState(g *Group, members []*Delivery) (string, string) {
count := map[State]int{} count := map[State]int{}
reached, ended := 0, 0 reached, ended := 0, 0
var uncovered []string var uncovered, lapsed []string
byID := map[string]*Delivery{}
for _, d := range members {
byID[d.ID] = d
}
for _, d := range members { for _, d := range members {
count[d.State]++
if d.State.Final() { if d.State.Final() {
ended++ ended++
} }
switch { switch {
case d.State.Reached(), d.State == Superseded && d.CoveredBy != "": case memberReached(d):
// Reached, whatever its final state: a failure or stop a later walk delivered since is no failure of the
// group (novox/hq issue 747).
reached++ reached++
continue
case stopLapsed(d, func(id string) *Delivery { return byID[id] }):
lapsed = append(lapsed, d.ID)
continue
case d.State == Superseded: case d.State == Superseded:
uncovered = append(uncovered, d.ID) uncovered = append(uncovered, d.ID)
} }
count[d.State]++
} }
n := len(members) n := len(members)
switch { switch {
@@ -2335,6 +2611,11 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
return "stopped", "a member was stopped" return "stopped", "a member was stopped"
case n > 0 && reached == n: case n > 0 && reached == n:
return "delivered", "every member is delivered" return "delivered", "every member is delivered"
case g.Closed && len(lapsed) > 0 && reached+len(lapsed) == n:
// Stopped for a member delivered since: its own change waits for a later walk of its repository, which then
// covers it (novox/hq issue 747).
return "delivering", fmt.Sprintf("%d of %d delivered; %s, stopped for a member delivered since, waits for a "+
"later walk of its repository", reached, n, strings.Join(lapsed, ", "))
case g.Closed && n > 0 && ended == n: case g.Closed && n > 0 && ended == n:
return "stopped", supersededWhy + strings.Join(uncovered, ", ") return "stopped", supersededWhy + strings.Join(uncovered, ", ")
case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" || case g.Closed && g.Check != nil && (g.Check.OutOfDate != "" || g.Check.Verdict == "" ||
@@ -0,0 +1,258 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 747: in the group fix/a-module-moves-only-when-its-program-changes, the controller's member #336
// (merged as 5d22d7df) failed at 22:32 during an outage of the forge's tools. The walk of the controller's next merge,
// #339 at 60cc9ec3, which contains 5d22d7df, failed with it and was resumed to done by a rebuild. mesh-delivery still
// held #336 failed, so its group stopped every later walk of the catalogue that carried the catalogue member's merge
// 2edef5a0: the catalogue could deliver nothing. A failed or stopped delivery is covered once a later finished walk of
// its repository that contains its merge built its modules, and a group stops a walk only for a member whose change is
// still undelivered.
//
// The shape is the live state of 2026-10-14 23:07, as mesh-delivery.show gave it, read back after a restart: the
// controller holds none of the walks of the night any more, only the walk of the catalogue's next merge.
const (
g747 = "fix/a-module-moves-only-when-its-program-changes"
ctl747, cat747 = "novox/mesh-controller", "novox/mesh-catalog"
head336, merge336 = "b34055f580d4e2a4", "5d22d7df86c73452"
head339, merge339 = "e4acab32d6a20ade", "60cc9ec363dfa7c6"
head301, merge301 = "d94764f00168471f", "2edef5a00ae05605"
merge307 = "0f76b6e2aaaaaaaa"
)
var (
at747 = time.Date(2026, 10, 14, 20, 21, 6, 0, time.UTC)
id336, id339, id301 = IDOf(ctl747, head336), IDOf(ctl747, head339), IDOf(cat747, head301)
unchangedRouteProxy747 = map[string]string{"route-proxy": "its build source holds none of the changed files"}
)
// night747 is the stuck state, with the forge saying, or not, that 60cc9ec3 contains 5d22d7df.
func night747(t *testing.T, contained bool) *world { return night747With(t, contained, nil) }
// night747With is night747 with #339's walk having built only the modules given, saying the others unchanged.
func night747With(t *testing.T, contained bool, built []string) *world {
t.Helper()
return night747Over(t, func(f *fakeForge) { f.contains[merge336+" in "+merge339] = contained }, built)
}
// night747Over is night747With with the forge set as given: one that is not told the pair cannot say.
func night747Over(t *testing.T, set func(*fakeForge), built []string) *world {
t.Helper()
at := at747
failedAt := at.Add(11 * time.Minute)
store := newMemStore()
moved339, unchanged339 := []string{"build-agent", "mesh-controller"}, unchangedRouteProxy747
if built != nil {
moved339, unchanged339 = built, map[string]string{}
for _, m := range []string{"build-agent", "mesh-controller", "route-proxy"} {
if !containsString(built, m) {
unchanged339[m] = "its build source holds none of the files #339 changed"
}
}
}
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Failed, Since: failedAt, Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"},
Plan: planMoving(ctl747, "build-agent", "mesh-controller", "route-proxy"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkFailed, Branch: "main", MovedSaid: true,
Note: "mesh-controller failed to build in tier 0", Created: at.Add(7 * time.Minute), Tiers: 2,
Unchanged: unchangedRouteProxy747},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m339 := &Delivery{ID: id339, Repository: ctl747, Commit: head339, Number: 339, Base: "main", Branch: "fix/726",
State: Failed, Since: failedAt, Created: at.Add(-10 * time.Minute), MergedAs: merge339, MergedAt: at.Add(7 * time.Minute),
AnsweredBy: merge339, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "mesh-controller"),
// Its walk failed with #336's, and a rebuild resumed it to done: the record it keeps says so.
Walk: &WalkSeen{ID: "plan-339", Commit: merge339, State: walkDone, Branch: "main", MovedSaid: true,
Moved: moved339, Unchanged: unchanged339,
Created: at.Add(11 * time.Minute), Tiers: 2, Tier: 2},
Transitions: []Transition{{At: failedAt, From: Delivering, To: Failed, Event: EvFailed,
Why: "its walk failed, carrying its own commit"}}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: failedAt, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"),
Transitions: []Transition{{At: failedAt, From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
for _, d := range []*Delivery{m336, m339, m301} {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
g := &Group{ID: g747, Members: []string{id301, id336}, Closed: true, Order: []string{id336, id301}, Said: "failed",
OrderOf: commitsOf([]*Delivery{m301, m336}), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "both compose", Commits: commitsOf([]*Delivery{m301, m336})}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
// The catalogue's next merge: its walk carries 2edef5a0, merged before it and never walked, and waits for its word.
ctlr.put(Walk{ID: "plan-cat", Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"gitea"}}, Modules: map[string]*WalkModule{"gitea": {}},
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge307}}}})
ctlr.mergedAt[merge301], ctlr.mergedAt[merge307] = m301.MergedAt, at.Add(46*time.Minute)
forge := newFakeForge()
set(forge)
w := newWorldOver(t, store, ctlr, forge, at.Add(47*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
func groupState747(w *world) (string, string) {
g := w.h.groups[g747]
return GroupState(g, w.h.membersOf(g))
}
// **A failed member is covered by a later finished walk that contains it**: #339's walk, done at 60cc9ec3, which the
// forge says contains 5d22d7df, built build-agent and mesh-controller, and #336's own walk said route-proxy unchanged.
// #336 stays failed, reads covered by that walk on its pull request, and its group no longer reads failed. #339, whose
// own walk was resumed to done, is covered by it too.
func TestAFailedMemberIsCoveredByALaterDoneWalkThatContainsIt(t *testing.T) {
w := night747(t, true)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "success" || !strings.Contains(desc, "covered after its failure") ||
!strings.Contains(desc, "plan-339") {
t.Fatalf("#336's status reads %s %q", state, desc)
}
if s := statusOn(w, ctl747, head336); !strings.HasPrefix(s, "success") {
t.Fatalf("the forge was told %q of #336", s)
}
if state, desc := statusOf(w.delivery(id339)); state != "success" || !strings.Contains(desc, "plan-339") {
t.Fatalf("#339 reads %s %q; want it covered by its own walk, resumed to done", state, desc)
}
if state, why := groupState747(w); state == "failed" || state == "stopped" {
t.Fatalf("the group reads %s: %s", state, why)
}
// Told once, and kept: a restart tells nothing again.
notes := len(w.forge.notesOn(merge336))
again := newWorldOver(t, w.store, w.ctl, w.forge, w.now.Add(time.Minute))
again.settleAll()
if state, _ := statusOf(again.delivery(id336)); len(w.forge.notesOn(merge336)) != notes || state != "success" {
t.Fatalf("a restart told it again or forgot it: %d notes, was %d", len(w.forge.notesOn(merge336)), notes)
}
}
// **The group then no longer stops the other repository's walk**: the catalogue member was stopped for #336 alone,
// so the walk of the catalogue's next merge, which carries its merge, is neither stopped nor held back, and the
// group waits for that walk to deliver the catalogue member's change.
func TestACoveredMemberNoLongerStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, true)
if len(w.ctl.withouts) > 0 || containsString(w.ctl.stops, "plan-cat") {
t.Fatalf("the catalogue's walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if stopped, _ := w.h.carriesStopped("plan-cat"); len(stopped) > 0 {
t.Fatalf("the catalogue's walk is held back for %v", stopped)
}
if state, why := groupState747(w); state != "delivering" || !strings.Contains(why, id301) {
t.Fatalf("the group reads %s: %s; want delivering, the catalogue member waiting for a later walk", state, why)
}
// Once a later walk of the catalogue that contains its merge built its modules, it is covered, and the group is
// delivered.
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk("plan-cat")
sent := w.now
r.Modules = map[string]*WalkModule{}
for _, m := range []string{"build-agent", "route-proxy", "gitea"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, "plan-cat") {
t.Fatalf("the catalogue member reads %s %q", state, desc)
}
if state, why := groupState747(w); state != "delivered" {
t.Fatalf("the group reads %s: %s", state, why)
}
}
// **A failed member not contained still stops it**: the forge says 60cc9ec3 does not contain 5d22d7df, so #336's change
// never reached the machines; it stays failed and uncovered, its group failed, and the catalogue's walk that carries
// the stopped member's merge is asked to leave it out.
func TestAFailedMemberNotContainedStillStopsTheOtherRepositorysWalk(t *testing.T) {
w := night747(t, false)
d := w.delivery(id336)
if state, desc := statusOf(d); d.State != Failed || state != "failure" {
t.Fatalf("#336 is %s and reads %s %q", d.State, state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("the catalogue's walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
}
// **A later walk that only said a module unchanged covers nothing of an earlier merge that moves it**: #339's walk
// built mesh-controller alone and said build-agent unchanged for #339's own files, which says nothing of #336's; only
// #336's own walk says that of its change (route-proxy). #336 stays failed and its group still stops the catalogue.
func TestALaterWalksUnchangedCoversNoModuleOfAnEarlierMerge(t *testing.T) {
w := night747With(t, true, []string{"mesh-controller"})
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q, though no walk built build-agent for it", state, desc)
}
if state, _ := groupState747(w); state != "failed" {
t.Fatalf("the group reads %s", state)
}
if len(w.ctl.withouts) == 0 {
t.Fatal("the catalogue's walk was not asked to leave the stopped merge out")
}
}
// **A forge that cannot say holds the walk only within the bound** (review of issue 747): while the forge errors on
// whether 60cc9ec3 contains 5d22d7df, the catalogue's walk is held back, loudly, neither stopped nor asked to leave the
// merge out, and #336 still reads failed. Past the bound the forge's silence holds nothing back any more: the walk
// is asked to leave the stopped merge out, as with no later walk at all.
func TestAForgeThatCannotSayHoldsTheWalkOnlyWithinTheBound(t *testing.T) {
for _, c := range []struct {
name string
set func(*fakeForge)
}{
{"erroring", func(f *fakeForge) { f.containsDown = true }},
{"not answering for the pair", func(*fakeForge) {}},
} {
t.Run(c.name, func(t *testing.T) {
w := night747Over(t, c.set, nil)
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
if state, desc := statusOf(w.delivery(id336)); state != "failure" {
t.Fatalf("#336 reads %s %q while the forge cannot say", state, desc)
}
said := ""
for _, l := range w.h.WalksWaiting() {
if l.Walk == "plan-cat" {
said += l.Says
}
}
w.later(3 * time.Minute) // past the grace an unheld walk is given, so `stalled` lists it
w.settleAll()
for _, l := range w.h.Stalled() {
if l.Walk == "plan-cat" {
said += " | " + l.Says
}
}
if strings.Count(said, "the forge is asked since") != 2 {
t.Fatalf("the hold is not said by `stalled walks` and `stalled`: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("within the bound the walk was not held: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
w.later(coverAskBound + time.Minute)
w.settleAll()
if len(w.ctl.withouts) == 0 || !strings.HasPrefix(w.ctl.withouts[0], "plan-cat without "+cat747+"@"+merge301) {
t.Fatalf("past the bound the walk was not asked to leave the stopped merge out: %v", w.ctl.withouts)
}
})
}
}
@@ -0,0 +1,292 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 753: once issue 747's fix was live (2026-10-14 23:35), the catalogue's walk plan-1792012015539062661 at
// 0f76b6e2 (#307's merge; build-agent, gitea, route-proxy to all four machines) was resumed and said "published; its
// walk waits for mesh-delivery's word: no delivery that has not ended follows it or is answered by it". Every delivery
// it carried had ended: #301 stopped by its group for the controller's #336, delivered since, and waiting for "a later
// walk of its repository that contains its merge"; #304 stopped; #307 delivered. The only walk that contained #301's
// merge waited for #301's word: a deadlock, released by a hand `plans go`. A delivery whose stop lapsed gives its word
// to the walk that answers its merge, and a walk with no living delivery behind it says what it carries.
//
// The shape is the live state as `mesh-delivery.show` and `mesh-controller.plans` gave it, read back after a restart.
const (
merge304, head304, head307 = "4c0ffee0bbbbbbbb", "a304a304a304a304", "b307b307b307b307"
plan753 = "plan-1792012015539062661"
)
var id304, id307 = IDOf(cat747, head304), IDOf(cat747, head307)
// night753 is the stuck walk: #336 delivered (its change reached the machines), #301 stopped by the given act, #304
// stopped by its group, #307 delivered, and the catalogue's walk at #307's merge naming all three merges, waiting.
func night753(t *testing.T, stop Transition) *world { return night753With(t, stop) }
// night753With is night753 with more deliveries kept: one in #301's group is ordered after #336 and before #301.
func night753With(t *testing.T, stop Transition, more ...*Delivery) *world {
t.Helper()
return night753Over(t, stop, nil, nil, more...)
}
// night753Over is night753With with more merges named by the walk and the controller set as given before the first
// tick.
func night753Over(t *testing.T, stop Transition, merges []WalkMerge, set func(*fakeController), more ...*Delivery) *world {
t.Helper()
at := at747
store := newMemStore()
m336 := &Delivery{ID: id336, Repository: ctl747, Commit: head336, Number: 336, Base: "main", Branch: g747, Group: g747,
State: Delivered, Since: at.Add(70 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge336, MergedAt: at,
AnsweredBy: merge336, Check: &Verdict{Gate: "pass"}, Plan: planMoving(ctl747, "build-agent", "mesh-controller"),
Walk: &WalkSeen{ID: "plan-336", Commit: merge336, State: walkDone, Branch: "main", MovedSaid: true,
Moved: []string{"build-agent", "mesh-controller"}, Created: at.Add(7 * time.Minute), Tiers: 2, Tier: 2}}
m301 := &Delivery{ID: id301, Repository: cat747, Commit: head301, Number: 301, Base: "main", Branch: g747, Group: g747,
State: Stopped, Since: stop.At, Created: at.Add(-2 * time.Hour), MergedAs: merge301, MergedAt: at.Add(7 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent", "route-proxy"), Transitions: []Transition{stop}}
if stop.By == byOwner {
m301.StoppedFor = id336
}
m304 := &Delivery{ID: id304, Repository: cat747, Commit: head304, Number: 304, Base: "main", Branch: "fix/forge-events",
State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge304,
MergedAt: at.Add(30 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "build-agent", "gitea", "route-proxy"),
// Stopped with its walk, which the walk of #307 took over, and covered since by the walk of #306 (issue 747).
DeliveredBy: "plan-1792011830935225963 (novox/mesh-catalog at 0361c44a into main, done, contains its merge 4c0ffee0)",
CoverTold: true,
Walk: &WalkSeen{ID: "plan-1792009301770286799", Commit: merge304, State: walkSuperseded, Branch: "main",
TakenOverBy: plan753, Waits: true, WaitedFor: "mesh-delivery", Created: at.Add(31 * time.Minute)},
Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Published, To: Stopped, Event: EvStop,
Why: "its walk plan-1792009301770286799 was stopped by mesh-delivery"}}}
m307 := &Delivery{ID: id307, Repository: cat747, Commit: head307, Number: 307, Base: "main", Branch: "fix/ts-check",
State: Delivered, Since: at.Add(90 * time.Minute), Created: at.Add(-time.Hour), MergedAs: merge307,
MergedAt: at.Add(46 * time.Minute), Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "build-agent")}
for _, d := range append([]*Delivery{m336, m301, m304, m307}, more...) {
if err := store.PutDelivery(d); err != nil {
t.Fatal(err)
}
}
inGroup, order := []*Delivery{m301, m336}, []string{id336}
for _, d := range more {
if d.Group == g747 {
inGroup, order = append(inGroup, d), append(order, d.ID)
}
}
order = append(order, id301)
var ids []string
for _, d := range inGroup {
ids = append(ids, d.ID)
}
g := &Group{ID: g747, Members: ids, Closed: true, Order: order, Said: "delivering",
OrderOf: commitsOf(inGroup), Check: &GroupCheck{Asked: "check-1", At: at.Add(-time.Minute),
Verdict: "pass", Summary: "all compose", Commits: commitsOf(inGroup)}}
if err := store.PutGroup(g); err != nil {
t.Fatal(err)
}
ctlr := newFakeController()
modules := map[string]*WalkModule{"build-agent": {}, "gitea": {}, "route-proxy": {}}
ctlr.put(Walk{ID: plan753, Repository: cat747, Branch: "main", Commit: merge307, Created: at.Add(46 * time.Minute),
State: walkBuilding, Tiers: [][]string{{"build-agent", "gitea", "route-proxy"}}, Modules: modules,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Merges: []WalkMerge{
{Repository: cat747, Commit: merge301, Carried: merge307}, {Repository: cat747, Commit: merge304, Carried: merge307},
{Repository: cat747, Commit: merge307}}}})
if len(merges) > 0 {
r := ctlr.walk(plan753)
r.Delivery.Merges = append(merges, r.Delivery.Merges...)
ctlr.put(r)
}
if set != nil {
set(ctlr)
}
ctlr.mergedAt[merge301], ctlr.mergedAt[merge304], ctlr.mergedAt[merge307] = m301.MergedAt, m304.MergedAt, m307.MergedAt
w := newWorldOver(t, store, ctlr, newFakeForge(), at.Add(100*time.Minute))
if err := w.h.Reconcile(); err != nil {
t.Fatal(err)
}
w.settleAll()
return w
}
// groupStop753 is #301's stop as its group made it: for #336, whose change was delivered since.
func groupStop753() Transition {
return Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}
}
func saidOf(lines []StalledLine, walk string) (string, State) {
said, state := "", State("")
for _, l := range lines {
if l.Walk == walk {
said, state = said+l.Says, l.State
}
}
return said, state
}
// **A stopped delivery gives its word to the later walk it waits for**: #301, stopped for #336, delivered since, waits for
// a later walk of the catalogue that contains its merge; plan-1792012015539062661 names that merge and waits for this
// owner's word. #301 gives it, once, saying why, and `stalled walks` never calls the walk unheld.
func TestAStoppedDeliveryGivesItsWordToTheWalkItWaitsFor(t *testing.T) {
w := night753(t, groupStop753())
if n := strings.Count(strings.Join(w.ctl.delivers, " "), plan753); n != 1 {
t.Fatalf("the walk was let go %d times (%v); want once", n, w.ctl.delivers)
}
d := w.delivery(id301)
if d.State != Stopped || !strings.Contains(d.LetGoWhy, lapsedWord(plan753)) || !strings.Contains(d.LetGoWhy, "issue 753") {
t.Fatalf("#301 is %s, its word given for %q", d.State, d.LetGoWhy)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state == Unheld {
t.Fatalf("the walk reads unheld: %q", said)
}
if len(w.ctl.withouts) > 0 || len(w.ctl.stops) > 0 {
t.Fatalf("the walk was stopped: without %v, stops %v", w.ctl.withouts, w.ctl.stops)
}
// Once that walk built build-agent and route-proxy and ended done, #301 is covered by it (issue 747).
w.forge.contains[merge301+" in "+merge307] = true
r := w.ctl.walk(plan753)
sent := w.now
for _, m := range []string{"build-agent", "gitea", "route-proxy"} {
r.Modules[m] = &WalkModule{State: "built", SentAt: &sent}
}
r.State = walkDone
w.h.WalkMoved(w.ctl.put(r))
w.later(time.Minute)
w.settleAll()
if state, desc := statusOf(w.delivery(id301)); state != "success" || !strings.Contains(desc, plan753) {
t.Fatalf("#301 reads %s %q", state, desc)
}
}
// **A walk still holding a live delivery not at its turn waits for it**: the stopped delivery's word is one turn among
// the walk's, never a way past a live delivery it carries.
func TestALapsedStopDoesNotLetGoPastALiveDelivery(t *testing.T) {
at := at747
held := &Delivery{ID: IDOf(cat747, "c308c308c308c308"), Repository: cat747, Commit: "c308c308c308c308", Number: 308,
Base: "main", Branch: "fix/held", State: Held, HeldWhy: "a person's", Since: at, Created: at.Add(-time.Hour),
MergedAs: "c308aaaaaaaaaaaa", MergedAt: at.Add(40 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Walk: &WalkSeen{ID: plan753, Commit: merge307, State: walkBuilding, Branch: "main",
Waits: true, Created: at.Add(46 * time.Minute)}}
w := night753With(t, groupStop753(), held)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go past a held delivery it carries: %v", w.ctl.delivers)
}
if said, state := saidOf(w.h.WalksWaiting(), plan753); state != Held {
t.Fatalf("`stalled walks` reads %s %q; want the held delivery", state, said)
}
}
// **A stop by a person gives no word**: #301 stopped by hand holds its change back; the walk is asked to leave it out,
// never let go for it.
func TestAPersonsStopGivesNoWord(t *testing.T) {
w := night753(t, Transition{At: at747.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: "jochen",
Why: "stopped by hand"})
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for a person's stop: %v", w.ctl.delivers)
}
if len(w.ctl.withouts) == 0 && len(w.ctl.stops) == 0 {
t.Fatal("the walk carrying a person's stop was neither left without it nor stopped")
}
}
// **A walk no living delivery holds says what it carries** (the issue's third expectation): with nothing to give it
// its word, `stalled walks` and, past the grace, `stalled` name every ended delivery it carries and its state, never
// only that nothing holds it.
func TestAWalkWithNoLivingDeliverySaysWhatItCarries(t *testing.T) {
w := night753(t, groupStop753())
// The walk read again as waiting, naming only #307's merge, which ended delivered: no delivery whose stop lapsed
// gives it its word, and no live one holds it.
r := w.ctl.walk(plan753)
r.Delivery.Go = nil
r.Delivery.Merges = []WalkMerge{{Repository: cat747, Commit: merge307}}
w.h.WalkMoved(w.ctl.put(r))
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state != Unheld || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled walks` reads %s %q; want unheld, naming %s (delivered)", state, said, id307)
}
w.settleAll()
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ = saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "every delivery it carries ended: ") || !strings.Contains(said, id307+" (delivered)") {
t.Fatalf("`stalled` reads %q", said)
}
}
// m299 is a member of #301's group ordered before it, stopped for #336 as #301 was, its merge on another walk.
func m299() *Delivery {
at := at747
return &Delivery{ID: IDOf(cat747, "e299e299e299e299"), Repository: cat747, Commit: "e299e299e299e299", Number: 299,
Base: "main", Branch: g747, Group: g747, State: Stopped, StoppedFor: id336, Since: at.Add(11 * time.Minute),
Created: at.Add(-3 * time.Hour), MergedAs: "e299aaaaaaaaaaaa", MergedAt: at.Add(5 * time.Second),
Check: &Verdict{Gate: "pass"}, Plan: planMoving(cat747, "gitea"),
Transitions: []Transition{{At: at.Add(11 * time.Minute), From: Ready, To: Stopped, Event: EvStop, By: byOwner,
Why: "its group " + g747 + " stopped: " + id336 + " failed"}}}
}
// **A stopped member gives its word only in its group's order** (review of issue 753): #299, ordered before #301 and
// stopped for #336 too, is not delivered and this walk does not carry its merge, so #301 gives no word — it would land
// before #299 — and `stalled walks` names #299. When the walk names #299's merge too, it delivers both, in order, and
// is let go.
func TestAStoppedMemberGivesItsWordOnlyInItsGroupsOrder(t *testing.T) {
w := night753With(t, groupStop753(), m299())
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go for #301 before #299, ordered before it: %v", w.ctl.delivers)
}
said, state := saidOf(w.h.WalksWaiting(), plan753)
if state == Unheld || !strings.Contains(said, m299().ID+" (stopped)") {
t.Fatalf("`stalled walks` reads %s %q; want #299 named", state, said)
}
w = night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: "e299aaaaaaaaaaaa", Carried: merge307}}, nil,
m299())
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying both members was not let go: %v", w.ctl.delivers)
}
}
// **A word the controller refuses is never silent** (review of issue 753): the controller keeps refusing it, and past
// the grace `stalled` lists the walk, with the refusal.
func TestARefusedWordIsListedByStalled(t *testing.T) {
w := night753Over(t, groupStop753(), nil, func(c *fakeController) { c.refuseDeliver = true })
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, state := saidOf(w.h.Stalled(), plan753)
if state != Published || !strings.Contains(said, "the controller refused its word: the controller refuses the word") {
t.Fatalf("`stalled` reads %s %q", state, said)
}
// Once the controller takes it, the walk is let go.
w.ctl.refuseDeliver = false
w.settleAll()
if !containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk was not let go once the controller took the word: %v", w.ctl.delivers)
}
}
// **A walk held back for another stopped change is never silent** (review of issue 753): #304 was stopped by a person,
// and the controller refuses to leave its merge out or to stop the walk, so the word #301 owes is never sent; past the
// grace `stalled` lists the walk, saying why.
func TestAWalkHeldForAPersonsStopIsListedByStalled(t *testing.T) {
at := at747
m305 := &Delivery{ID: IDOf(cat747, "f305f305f305f305"), Repository: cat747, Commit: "f305f305f305f305", Number: 305,
Base: "main", Branch: "fix/other", State: Stopped, Since: at.Add(80 * time.Minute), Created: at.Add(-time.Hour),
MergedAs: "f305aaaaaaaaaaaa", MergedAt: at.Add(35 * time.Minute), Check: &Verdict{Gate: "pass"},
Plan: planMoving(cat747, "gitea"), Transitions: []Transition{{At: at.Add(80 * time.Minute), From: Ready,
To: Stopped, Event: EvStop, By: "jochen", Why: "stopped by hand"}}}
w := night753Over(t, groupStop753(), []WalkMerge{{Repository: cat747, Commit: m305.MergedAs, Carried: merge307}},
func(c *fakeController) { c.refuseWithout, c.refuseStop = true, true }, m305)
if containsString(w.ctl.delivers, plan753) {
t.Fatalf("the walk carrying a person's stop was let go: %v", w.ctl.delivers)
}
w.later(matchGrace + time.Minute)
w.settleAll()
said, _ := saidOf(w.h.Stalled(), plan753)
if !strings.Contains(said, "owe it (novox/hq issue 753)") || !strings.Contains(said, "f305aaaaaaaaaaaa") ||
!strings.Contains(said, "whose delivery was stopped") {
t.Fatalf("`stalled` reads %q", said)
}
}
@@ -0,0 +1,151 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 783: mesh-catalog commit b5417c28 was pushed while #305 (from branch adr-359-mesh-mail-733) was
// open, so this owner recorded its delivery as #305's. #305 then closed unmerged (its base branch had been
// deleted), and #315, opened from the same branch and head, merged it as 0914a33f. The delivery stayed stopped
// ("the pull request closed unmerged"), though it recorded merged_as 0914a33f: a rule did not hold — a delivery
// keyed by commit follows the pull request that merges that commit, and a closed duplicate does not end it — and
// the walk that carried the merge waited 64 minutes for this owner's word until a person's `plans go` released it.
//
// A delivery stopped only because its pull request closed unmerged, with no merge recorded, does not end its
// commit's delivery for another pull request with the same head: it is started over, under the same id, for the
// pull request that announces or merges it next.
const (
head783, merge783 = "d305d305d305d305", "d315d315d315d315"
head783b, merge783b = "d306d306d306d306", "d316d316d316d316"
head783c = "d307d307d307d307"
head783d, merge783d = "d308d308d308d308", "d318d318d318d318"
catalog783 = "novox/mesh-catalog"
branch783 = "adr-359-mesh-mail-733"
)
// TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit is the issue's own shape: #305
// announced, closed unmerged (stopped); #315 announced from the same head, checked and readied; #315 merges it,
// and the walk that answers that merge is let go — the delivery follows #315, never stuck on #305's stop.
func TestADeliveryStoppedByAClosedDuplicateFollowsThePullRequestThatMergesItsCommit(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783)
w.h.PullUpdated(pr(catalog783, 305, head783, branch783, "src/a.go"))
w.h.PullClosed(pr(catalog783, 305, head783, branch783))
if w.state(id) != Stopped {
t.Fatalf("#305 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 315, head783, branch783, "src/a.go"))
if d := w.delivery(id); d.State == Stopped || d.Number != 315 {
t.Fatalf("#315, the same head, left it %s, Number %d; want started over for #315, not stopped", d.State, d.Number)
}
if d := w.delivery(id); !strings.Contains(d.Transitions[len(d.Transitions)-1].Why, "#305 with the same head closed unmerged") {
t.Fatalf("the start-over's transition names what it replaces: %q", d.Transitions[len(d.Transitions)-1].Why)
}
w.h.Checked(verdict(catalog783, 315, head783, "pass"))
if d := w.delivery(id); d.State != Ready {
t.Fatalf("a passing verdict for #315 left it %s", d.State)
}
walk := w.ctl.put(aWalk("plan-783", catalog783, merge783, true))
w.h.WalkMoved(walk)
w.h.PullMerged(merged(catalog783, 315, head783, merge783))
if d := w.delivery(id); d.State != Published || d.Number != 315 {
t.Fatalf("#315's merge left it %s, Number %d; want published under #315", d.State, d.Number)
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783" {
t.Fatalf("the walk that answers #315's merge was not let go: %v", w.ctl.delivers)
}
if d := w.delivery(id); d.State != Delivering {
t.Fatalf("let go, it is %s; want delivering", d.State)
}
}
// TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo is decision 4: the merging pull request is never
// announced on its own (heard only as `pull.merged`) — the start-over happens in PullMerged itself, not only in
// PullUpdated, so a merge heard without the open announcement is not lost either.
func TestAMergeWithoutAnAnnouncementStartsOverAStoppedDuplicateToo(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783b)
w.h.PullUpdated(pr(catalog783, 306, head783b, branch783+"-b", "src/b.go"))
w.h.PullClosed(pr(catalog783, 306, head783b, branch783+"-b"))
if w.state(id) != Stopped {
t.Fatalf("#306 closed unmerged left it %s; want stopped", w.state(id))
}
walk := w.ctl.put(aWalk("plan-783b", catalog783, merge783b, true))
w.h.WalkMoved(walk)
m := merged(catalog783, 316, head783b, merge783b)
m.HeadChecks = map[string]string{"mesh/merge-gate": "success", "mesh/repo-check": "success"}
w.h.PullMerged(m)
if d := w.delivery(id); d.State == Stopped || d.Number != 316 {
t.Fatalf("#316's merge, heard with no announcement of its own, left it %s, Number %d; want started over for #316",
d.State, d.Number)
}
w.settleAll()
if len(w.ctl.delivers) != 1 || w.ctl.delivers[0] != "plan-783b" {
t.Fatalf("the walk that answers #316's merge was not let go: %v", w.ctl.delivers)
}
}
// TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped guards decision 3's "any other stopped
// delivery ... is untouched": the same pull request announced again after its own close is the ordinary
// 'announced again' path, never a start-over, and stays stopped.
func TestAStoppedDeliveryAnnouncedAgainByItsOwnPullRequestStaysStopped(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783c)
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
w.h.PullClosed(pr(catalog783, 307, head783c, branch783+"-c"))
if w.state(id) != Stopped {
t.Fatalf("#307 closed unmerged left it %s; want stopped", w.state(id))
}
w.h.PullUpdated(pr(catalog783, 307, head783c, branch783+"-c", "src/c.go"))
if d := w.delivery(id); d.State != Stopped || d.Number != 307 {
t.Fatalf("#307 announced again by itself left it %s, Number %d; want it to stay stopped", d.State, d.Number)
}
}
// TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge is review round 1's
// decision: a delivery started over from a merge heard without its open announcement (as in decision 4, above)
// takes FromForge exactly as a first-heard merge does, so one replayed from the bus's history long after it
// happened still falls under the history rule (novox/hq issue 313) and is never held for a person.
func TestAMergeHeardLongAfterStartingOverAStoppedDuplicateIsHistoryLikeAnyOtherReplayedMerge(t *testing.T) {
w := newWorld(t)
id := IDOf(catalog783, head783d)
w.h.PullUpdated(pr(catalog783, 308, head783d, branch783+"-d", "src/d.go"))
w.h.PullClosed(pr(catalog783, 308, head783d, branch783+"-d"))
if w.state(id) != Stopped {
t.Fatalf("#308 closed unmerged left it %s; want stopped", w.state(id))
}
m := merged(catalog783, 318, head783d, merge783d)
m.MergedAt = w.now.Add(-historyLate - time.Minute).UTC().Format(time.RFC3339)
w.h.PullMerged(m)
d := w.delivery(id)
if d.State == Stopped || d.Number != 318 {
t.Fatalf("#318's merge, heard long after it happened with no announcement of its own, left it %s, Number %d; "+
"want started over for #318", d.State, d.Number)
}
if !d.FromForge {
t.Fatal("a merge heard without its own announcement is made from the forge's word: FromForge is false")
}
if d.History == "" {
t.Fatalf("a merge heard %s after it happened is history, never held for a person; History is empty", historyLate+time.Minute)
}
if d.State == Held {
t.Fatalf("history is never held for a person; state is %s, HeldWhy %q", d.State, d.HeldWhy)
}
}
@@ -83,6 +83,14 @@ type Delivery struct {
// CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note // CoverTold is whether the forge was told that such a stop was covered since: its status, its view and its note
// (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too. // (novox/hq issue 480). Kept, so a restart tells it once, and a stop covered before this was built is told too.
CoverTold bool `json:"cover_told,omitempty"` CoverTold bool `json:"cover_told,omitempty"`
// DeliveredBy says, for a delivery that failed or was stopped, the later finished walk of its repository whose
// commit contains its merge and which built every module it moves — each it did not build, its own walk said
// unchanged: its change reached the machines after all (novox/hq issue 747). Its state stays final, as a covered
// stop's does (issue 480), and it reads covered: its group counts it as delivered.
DeliveredBy string `json:"delivered_by,omitempty"`
// StoppedFor is, for a member its group stopped, the member whose end stopped it (novox/hq issue 747): once that
// member's change reached the machines, the stop has lapsed, and holds no walk of its repository back.
StoppedFor string `json:"stopped_for,omitempty"`
// TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it, // TurnAwaits says, for a group member whose turn waits for a later merge to carry a superseded member before it,
// which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9, // which merges and until when; TurnAwaitsUntil is that moment, which bounds the wait (ADR 0276 decision 9,
// novox/hq issue 445). `show` shows both, and `stalled` says them past that moment. // novox/hq issue 445). `show` shows both, and `stalled` says them past that moment.
@@ -303,18 +311,77 @@ func (d *Delivery) FromForgesWord() bool {
// shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose // shownOnItsPullRequest is whether its view and status are kept on its pull request: not for history, whose
// pull request the forge may no longer hold — or holds under its number another one. // pull request the forge may no longer hold — or holds under its number another one.
// coveredStop is whether a delivery stopped because its walk ended done before its word without building its modules // coveredStop is whether a delivery that ended stopped or failed was covered since, and reads covered. A stop
// (novox/hq issue 474) was covered since: a later merge into its trunk that contains it was delivered, carrying every // because its walk ended done before its word without building its modules (novox/hq issue 474) is covered when a
// module it moves to every machine running it (ADR 0270's cover test, kept as CoveredBy). Its state stays stopped — // later merge into its trunk that contains it was delivered, carrying every module it moves to every machine running
// final, as a covered superseded member stays superseded (issue 442) — and it reads covered, by that merge (novox/hq // it (ADR 0270's cover test, kept as CoveredBy; issue 480). Any stop or failure is covered when a later finished walk
// issue 480). A stop for any other reason never reads covered. // of its repository whose commit contains its merge built its modules (DeliveredBy, novox/hq issue 747). Its state
// stays final, as a covered superseded member stays superseded (issue 442), and it reads covered, by that merge or
// walk. A person's stop with a cover found only while it was held keeps its error.
func (d *Delivery) coveredStop() bool { func (d *Delivery) coveredStop() bool {
return d.State == Stopped && d.Stranded != "" && d.CoveredBy != "" switch d.State {
case Stopped:
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
case Failed:
return d.DeliveredBy != ""
}
return false
} }
// coveredStopSaid is how a covered stop reads: on the forge's status, in its view and note, and in `show`. // coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
func (d *Delivery) coveredStopSaid() string { func (d *Delivery) coveredStopSaid() string {
return "covered after its stop: a later merge that contains it was delivered: " + d.CoveredBy ended := "its stop"
if d.State == Failed {
ended = "its failure"
}
if d.Stranded != "" && d.CoveredBy != "" {
return "covered after " + ended + ": a later merge that contains it was delivered: " + d.CoveredBy
}
return "covered after " + ended + ": a later walk that contains it was delivered: " + d.DeliveredBy
}
// memberReached is whether a group counts a member as delivered: its change reached the machines, or had nothing to
// reach them with — delivered, history or covered, a superseded member a later delivered walk carried (issue 442),
// or a stop or failure covered since (issues 480 and 747).
func memberReached(d *Delivery) bool {
return d.State.Reached() || d.State == Superseded && d.CoveredBy != "" || d.coveredStop()
}
// stoppedFor is the member whose end stopped this one in its group: kept since novox/hq issue 747, and read from the
// stop's reason for a stop made before ("its group <id> stopped: <member> …").
func (d *Delivery) stoppedFor() string {
if d.StoppedFor != "" || d.State != Stopped || d.Group == "" {
return d.StoppedFor
}
prefix := "its group " + d.Group + " stopped: "
for i := len(d.Transitions) - 1; i >= 0; i-- {
t := d.Transitions[i]
if t.Event != EvStop {
continue
}
if t.By != byOwner || !strings.HasPrefix(t.Why, prefix) {
return ""
}
member, _, _ := strings.Cut(strings.TrimPrefix(t.Why, prefix), " ")
return strings.TrimRight(member, ",;:")
}
return ""
}
// stopLapsed is whether a member its group stopped for another member is stopped for nothing any more: that member's
// change reached the machines since — delivered, or its failure covered by a later walk (novox/hq issue 747). Its own
// change is still to deliver, by a later walk of its repository that contains it; its stop holds none back. find
// gives a delivery by its id.
func stopLapsed(d *Delivery, find func(string) *Delivery) bool {
if d.State != Stopped || d.coveredStop() {
return false
}
by := d.stoppedFor()
if by == "" || by == d.ID {
return false
}
o := find(by)
return o != nil && memberReached(o)
} }
func (d *Delivery) shownOnItsPullRequest() bool { func (d *Delivery) shownOnItsPullRequest() bool {
@@ -72,6 +72,10 @@ func (h *Holder) waitsFor(d *Delivery) string {
return "its walk: " + clip(d.Walk.Note, 120) return "its walk: " + clip(d.Walk.Note, 120)
} }
case Stopped: case Stopped:
if h.stopLapsed(d) {
return "a later walk of its repository that contains its merge: its group stopped it for " + d.stoppedFor() +
", whose change was delivered since (novox/hq issue 747)"
}
if strandedUncovered(d) { if strandedUncovered(d) {
return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " + return "a later delivered merge to carry " + strings.Join(d.notMovedBy(), ", ") + ": it is stopped, and " +
"listed by stalled until one does (novox/hq issue 480)" "listed by stalled until one does (novox/hq issue 480)"
+1 -1
View File
@@ -21,7 +21,7 @@ operator, it is accepted only on the operator's Approve, given on a channel that
| `links` | `number`, `kind` | the record's typed links, out and in | | `links` | `number`, `kind` | the record's typed links, out and in |
| `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line | | `history` | `number`, `kind`, `from`, `to`, `bound` | every version; with `from` and `to`, the difference between the two, line by line |
| `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted | | `reserve` | `kind`, `for` | the next decision or research number, asked of the issue tracker (`issue-tracker.reserve`); spent whether or not its record is accepted |
| `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`). Each version proposed is then reviewed by the knowledge base itself (below) | | `propose` | the record whole | proposed, or refused naming every rule it breaks; an accepted record changes only by a progressive insight (`insight`), but an active research effort takes new versions freely, each reviewed as a proposal. Each version proposed is then reviewed by the knowledge base itself (below) |
| `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number | | `import` | `dry-run`, `commit` | hq's decision records and research efforts at the commit the records module stands at, each whole at its own number |
| `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number | | `return` | `number`, `reason` | drafted clauses sent back to be drafted again; refused for a record whose clauses another record changed or links by number |
@@ -0,0 +1,150 @@
package main
import (
"strings"
"testing"
"time"
)
// effortAsk is a research effort proposal with the given documents.
func effortAsk(n int, status string, docs ...Section) ProposeAsk {
return ProposeAsk{Kind: KindResearch, Number: n, Title: "A vendor-neutral inference layer", Date: "2026-10-11",
Deciders: []string{"jochen"}, Status: status, Sections: docs,
Links: []LinkAsk{{Kind: LinkTouches, To: "0290"}}}
}
// novox/hq issue 844: an active research effort takes new versions freely, each reviewed as a proposal.
func TestAnActiveEffortTakesANewVersionWithANewDocument(t *testing.T) {
w := imported(t, newMemStore())
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
if err != nil {
t.Fatal(err)
}
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
key := Key{KindResearch, r.Number}
reviews := len(w.judge.reviews)
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
next.Why = "findings added"
if _, err := w.k.Propose("author@g14", next); err != nil {
t.Fatalf("an active effort refused a new version: %v", err)
}
e := mustGet(t, w.k.Store, key)
if e.Pending == nil || e.Current.Section("01-findings.md") != "" {
t.Fatalf("the new version was served before its review")
}
w.reviewAll(t)
q := w.judge.reviews[reviews]
if strings.Contains(q, "a correction of fact (a progressive insight)") {
t.Fatalf("the review judged a new version as an insight:\n%.400s", q)
}
e = mustGet(t, w.k.Store, key)
if e.Pending != nil || e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "A new finding." ||
len(e.Current.Insights) != 0 {
t.Fatalf("after the review: %s, pending %v, insights %v", e.Current.Status, e.Pending != nil, e.Current.Insights)
}
h, err := w.k.History(key, 0, 0, 0)
if err != nil {
t.Fatal(err)
}
for _, v := range h.Versions {
if v.Insight {
t.Errorf("version %d is marked an insight", v.N)
}
}
}
func TestAGraduatedEffortAndADecisionStillTakeOnlyAnInsight(t *testing.T) {
w := imported(t, newMemStore())
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
if err != nil {
t.Fatal(err)
}
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
grad := effortAsk(r.Number, StatusGraduated, overview)
grad.Links = append(grad.Links, LinkAsk{Kind: LinkBecame, To: "0290"})
w.accepted(t, "author@g14", grad)
next := effortAsk(r.Number, StatusGraduated, overview, Section{Name: "01-findings.md", Text: "More."})
next.Links = grad.Links
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "progressive insight") {
t.Fatalf("a graduated effort took a plain version: %v", err)
}
n := w.reserved(t, "author@g14")
w.accepted(t, "author@g14", valid(n))
p := valid(n)
p.Sections[2].Text = "Changed."
if _, err := w.k.Propose("author@g14", p); err == nil || !strings.Contains(err.Error(), "progressive insight") {
t.Fatalf("a decision took a plain version: %v", err)
}
}
// An old active effort's new text is held to the rules as of the day it is proposed, not the day the effort began.
func TestANewVersionOfAnOldActiveEffortIsHeldToTodaysRules(t *testing.T) {
w := imported(t, newMemStore())
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
if err != nil {
t.Fatal(err)
}
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
first := effortAsk(r.Number, StatusActive, overview)
first.Date = "2026-10-01"
w.accepted(t, "author@g14", first)
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "The control plane wrote them."})
next.Date = "2026-10-01"
if _, err := w.k.Propose("author@g14", next); err == nil || !strings.Contains(err.Error(), "retired-word") {
t.Fatalf("a retired word in new text of an old effort: %v", err)
}
}
func TestAChangesVerdictLeavesTheServedVersionServedAndGraduatingNeedsABecameLink(t *testing.T) {
w := imported(t, newMemStore())
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an effort")
if err != nil {
t.Fatal(err)
}
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be."}
w.accepted(t, "author@g14", effortAsk(r.Number, StatusActive, overview))
key := Key{KindResearch, r.Number}
w.judge.review = `{"verdict":"changes","findings":["01-findings.md: say where the finding comes from"]}`
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
if _, err := w.k.Propose("author@g14", next); err != nil {
t.Fatal(err)
}
w.reviewAll(t)
e := mustGet(t, w.k.Store, key)
if e.Pending == nil || e.Pending.Reviews[len(e.Pending.Reviews)-1].Verdict != VerdictChanges ||
e.Current.Status != StatusActive || e.Current.Section("01-findings.md") != "" {
t.Fatalf("after changes: pending %v, current %+v", e.Pending != nil, e.Current.Sections)
}
grad := effortAsk(r.Number, StatusGraduated, overview)
if _, err := w.k.Propose("author@g14", grad); err == nil || !strings.Contains(err.Error(), "became") {
t.Fatalf("graduated without a became link: %v", err)
}
}
// An earlier marked insight in an old effort's text does not predate the effort, and a new version is accepted.
func TestAnOldActiveEffortWithAMarkedInsightTakesANewVersion(t *testing.T) {
w := imported(t, newMemStore())
w.clock = time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC)
r, err := w.k.ReserveNumber("author@g14", KindResearch, "an old effort")
if err != nil {
t.Fatal(err)
}
overview := Section{Name: "00-overview.md", Text: "What an inference layer would be.\n\n**Progressive insight — 2026-10-05.** A fact was corrected."}
first := effortAsk(r.Number, StatusActive, overview)
first.Date = "2026-10-01"
w.accepted(t, "author@g14", first)
w.clock = time.Date(2026, 10, 15, 9, 0, 0, 0, time.UTC)
next := effortAsk(r.Number, StatusActive, overview, Section{Name: "01-findings.md", Text: "A new finding."})
next.Date = "2026-10-01"
if _, err := w.k.Propose("author@g14", next); err != nil {
t.Fatalf("a new version with an earlier marked insight: %v", err)
}
w.reviewAll(t)
e := mustGet(t, w.k.Store, Key{KindResearch, r.Number})
if e.Pending != nil || e.Current.Section("01-findings.md") != "A new finding." || e.Current.Date != "2026-10-01" {
t.Fatalf("after the review: pending %v, date %s", e.Pending != nil, e.Current.Date)
}
}
@@ -357,7 +357,7 @@ func (k *KB) Get(a GetAsk) (*Got, error) {
g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor) g.Reserved = fmt.Sprintf("reserved by %s at %s for %q", r.ReservedBy, r.ReservedAt.Format("2006-01-02 15:04 UTC"), r.ReservedFor)
} }
if e.Pending != nil && a.Version == 0 { if e.Pending != nil && a.Version == 0 {
g.Pending = fmt.Sprintf("version %d, a progressive insight, waits for its review: history shows what it changes", e.PendingVersion) g.Pending = fmt.Sprintf("version %d, a progressive insight or a new version of an active effort, waits for its review: history shows what it changes", e.PendingVersion)
} }
if r.Ask != nil { if r.Ask != nil {
g.Waiting = "the operator's ask: " + r.Ask.Said g.Waiting = "the operator's ask: " + r.Ask.Said
@@ -335,7 +335,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+ return nil, fmt.Errorf("%s has a progressive insight by %s waiting for its review: only its author, proven by the "+
"runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller) "runtime, revises it (this call is by %s). Nothing was proposed", key, held.Pending.Author, caller)
} }
if strings.TrimSpace(a.Insight) == "" { // An active research effort is still being researched: it takes new versions freely, each reviewed as a
// proposal while the served version stays (novox/hq issue 844). Every other accepted record needs an insight.
revision := a.Kind == KindResearch && held.Current.Status == StatusActive && strings.TrimSpace(a.Insight) == ""
if strings.TrimSpace(a.Insight) == "" && !revision {
return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+ return nil, fmt.Errorf("%s is %s: it changes only by a new record, or by a progressive insight, a correction of fact "+
"that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status) "that leaves the decision standing (give insight: what stood before). Nothing was proposed", key, held.Current.Status)
} }
@@ -349,8 +352,14 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}} r.Pending = &PendingState{Insight: Insight{Date: k.now().Format("2006-01-02"), Before: strings.TrimSpace(a.Insight), By: caller}}
r.Date = held.Current.Date r.Date = held.Current.Date
r.Effort = a.Status r.Effort = a.Status
if revision {
r.Pending = &PendingState{}
}
if why == "" { if why == "" {
why = "a progressive insight proposed" why = "a progressive insight proposed"
if revision {
why = "a new version proposed"
}
} }
default: default:
return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status) return nil, fmt.Errorf("%s is %s and takes no proposal; nothing was proposed", key, held.Working().Status)
@@ -370,7 +379,13 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
if !current && a.Kind == KindResearch && r.Effort == "" { if !current && a.Kind == KindResearch && r.Effort == "" {
r.Effort = held.Current.Status r.Effort = held.Current.Status
} }
refusals = append(refusals, JudgeRecord(r, held, k.lookup(ctx), ref)...) // A new version of an active effort is new text: the dated rules hold it from the day it is proposed, while the
// record keeps its own date (novox/hq issue 844).
day := r.Date
if r.Pending != nil && !r.IsInsight() {
day = k.now().Format("2006-01-02")
}
refusals = append(refusals, JudgeRecordOn(r, held, k.lookup(ctx), ref, day)...)
if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes { if raw, _ := json.Marshal(r); len(raw) > MostRecordBytes {
refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)}) refusals = append(refusals, Refusal{Rule: "size", Says: fmt.Sprintf("the record is %d bytes; a record holds at most %d", len(raw), MostRecordBytes)})
} }
@@ -381,7 +396,7 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
} }
return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- ")) return nil, fmt.Errorf("%s is refused by %d rule(s), and nothing was written:\n- %s", key, len(refusals), strings.Join(lines, "\n- "))
} }
v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: !current} v := Version{By: caller, At: k.now(), Why: why, Record: r, Insight: r.IsInsight()}
if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil { if err := k.Store.Save(ctx, v, current, false, held.Revision); err != nil {
return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err) return nil, fmt.Errorf("%s could not be written (%v); nothing was proposed", key, err)
} }
@@ -397,7 +412,10 @@ func (k *KB) Propose(caller string, a ProposeAsk) (*Proposed, error) {
if r.NeedsOperator { if r.NeedsOperator {
says += ", and then the operator's Approve" says += ", and then the operator's Approve"
} }
if !current { if !current && !r.IsInsight() {
says = fmt.Sprintf("A new version of %s is proposed as version %d. The knowledge base reviews it now; the served version stays "+
"until that review approves it", key, n)
} else if !current {
says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+ says = fmt.Sprintf("A progressive insight to %s is proposed as version %d. The knowledge base reviews it now; the accepted "+
"version is served until that review approves it", key, n) "version is served until that review approves it", key, n)
} }
@@ -416,8 +434,11 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
at := k.now() at := k.now()
if insight { if insight {
in := r.Pending.Insight in := r.Pending.Insight
isInsight := r.IsInsight()
r.Pending = nil r.Pending = nil
r.Insights = append(r.Insights, in) if isInsight {
r.Insights = append(r.Insights, in)
}
r.Status = held.Current.Status r.Status = held.Current.Status
if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded { if r.Kind == KindResearch && contains(ResearchStatuses, r.Effort) && held.Current.Status != StatusSuperseded {
r.Status = r.Effort r.Status = r.Effort
@@ -442,7 +463,7 @@ func (k *KB) accept(ctx context.Context, held *Entry, w *Record, by, why, warran
if warrant != "" { if warrant != "" {
why += "; " + warrant why += "; " + warrant
} }
if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight}, true, true, held.Revision); err != nil { if err := k.Store.Save(ctx, Version{By: by, At: at, Why: why, Record: r, Insight: insight && w.IsInsight()}, true, true, held.Revision); err != nil {
return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err) return nil, fmt.Errorf("%s could not be accepted (%v); it stays proposed", r.Key(), err)
} }
k.emit(EventAccepted, r, by, why, held.Versions+1) k.emit(EventAccepted, r, by, why, held.Versions+1)
@@ -917,7 +917,7 @@ func Tools(k *KB, caller func(map[string]any) string) []stdio.Tool {
"links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " + "links (supersedes, changes \"0276 §4\", extends, became, cites), and whether it needs the operator, with the brief its ask shows. Refused, naming " +
"every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " + "every reason at once and writing nothing, when it breaks one of the 30 rules that judge one record or its prose — a " +
"link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " + "link to nothing, a clause without a bold lead, a superseded record cited alone, an unknown topic, a retired word. A " +
"proposal is revised by proposing it again; an accepted record changes only by a progressive insight (insight: what " + "proposal is revised by proposing it again; an accepted record (an active research effort excepted, which takes new versions) changes only by a progressive insight (insight: what " +
"stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " + "stood before). The knowledge base then reviews each version itself, in a judgement that sees only the record, the " +
"records it links and the rules. A decision that names no home and that no accepted record links is not judged: " + "records it links and the rules. A decision that names no home and that no accepted record links is not judged: " +
"its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " + "its review asks changes for the whole-set rule cited, which its acceptance would break. On approve it is accepted (a record that needs the operator waits for the operator's " +
@@ -192,7 +192,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
if err != nil || e == nil { if err != nil || e == nil {
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err) return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
} }
if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil}, if err := k.Store.Save(ctx, Version{By: ReviewBy, At: k.now(), Why: said, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
e.Pending == nil, false, e.Revision); err != nil { e.Pending == nil, false, e.Revision); err != nil {
return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err) return "", fmt.Errorf("%s: %s, and that could not be kept (%v)", w.Key(), said, err)
} }
@@ -208,7 +208,7 @@ func (k *KB) askOperator(ctx context.Context, held *Entry, w *Record, by string,
return unasked(fmt.Sprintf("the ask could not be made (%v)", err)) return unasked(fmt.Sprintf("the ask could not be made (%v)", err))
} }
w.Ask = oa w.Ask = oa
if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil}, if err := k.Store.Save(ctx, Version{By: by, At: k.now(), Why: "reviewed: approved; the operator is asked", Record: w, Insight: held.Pending != nil && held.Pending.IsInsight()},
held.Pending == nil, false, held.Revision); err != nil { held.Pending == nil, false, held.Revision); err != nil {
return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err) return "", fmt.Errorf("%s: the review could not be kept (%v); the operator was not asked", w.Key(), err)
} }
@@ -267,7 +267,7 @@ func (k *KB) Reconcile() ([]string, error) {
func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string { func (k *KB) endAsk(ctx context.Context, e *Entry, w *Record, why string) string {
w = w.Clone() w = w.Clone()
w.Ask.Said = why w.Ask.Said = why
if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil}, if err := k.Store.Save(ctx, Version{By: "the knowledge base", At: k.now(), Why: "the operator's ask " + why, Record: w, Insight: e.Pending != nil && e.Pending.IsInsight()},
e.Pending == nil, false, e.Revision); err != nil { e.Pending == nil, false, e.Revision); err != nil {
return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err) return fmt.Sprintf("%s: the end of its ask could not be kept (%v)", w.Key(), err)
} }
@@ -360,6 +360,10 @@ type PendingState struct {
Insight Insight `json:"insight"` Insight Insight `json:"insight"`
} }
// IsInsight says the record is waiting as a progressive insight, a correction of fact that says what stood before.
// A pending version without one is a new version of an active research effort (novox/hq issue 844).
func (r *Record) IsInsight() bool { return r.Pending != nil && r.Pending.Insight.Before != "" }
// Key is the record's key. // Key is the record's key.
func (r *Record) Key() Key { return Key{r.Kind, r.Number} } func (r *Record) Key() Key { return Key{r.Kind, r.Number} }
@@ -111,7 +111,7 @@ var reviewSchema = map[string]any{
func ReviewQuestion(r *Record, linked []*Record) string { func ReviewQuestion(r *Record, linked []*Record) string {
var b strings.Builder var b strings.Builder
what := "a proposed " + r.Kind + " record" what := "a proposed " + r.Kind + " record"
if r.Pending != nil { if r.IsInsight() {
what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before what = "a correction of fact (a progressive insight) proposed to an accepted " + r.Kind + " record; what stood before: " + r.Pending.Insight.Before
} }
fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+ fmt.Fprintf(&b, "You review %s of the mesh's knowledge base, %s. You see only the record, the records it links, and the "+
@@ -475,7 +475,7 @@ func (k *KB) reviewOne(key Key) (string, error) {
version = now.PendingVersion version = now.PendingVersion
} }
save := func(r *Record, why string) error { save := func(r *Record, why string) error {
return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil}, return k.Store.Save(ctx, Version{By: by, At: k.now(), Why: why, Record: r, Insight: now.Pending != nil && now.Pending.IsInsight()},
now.Pending == nil, false, now.Revision) now.Pending == nil, false, now.Revision)
} }
if err != nil { if err != nil {
@@ -123,9 +123,16 @@ func allText(r *Record) string {
// JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the // JudgeRecord holds a proposed record to the 30 rules. held is the entry its number holds (a reservation, the
// proposal being revised, or the accepted record an insight corrects); lookup reads every other record. // proposal being revised, or the accepted record an insight corrects); lookup reads every other record.
func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal { func JudgeRecord(r *Record, held *Entry, lookup Lookup, ref *Reference) []Refusal {
return JudgeRecordOn(r, held, lookup, ref, r.Date)
}
// JudgeRecordOn is JudgeRecord with the day the dated rules are held from: the record's own day for a new
// record, the day of the proposal for a new version of an active research effort (novox/hq issue 844). The rules
// that read the record's date (the date and insight-form rules) keep reading r.Date.
func JudgeRecordOn(r *Record, held *Entry, lookup Lookup, ref *Reference, day string) []Refusal {
var out []Refusal var out []Refusal
refuse := func(rule, format string, a ...any) { refuse := func(rule, format string, a ...any) {
if Holds(rule, r.Kind, r.Date) { if Holds(rule, r.Kind, day) {
out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)}) out = append(out, Refusal{Rule: rule, Says: fmt.Sprintf(format, a...)})
} }
} }
+40
View File
@@ -0,0 +1,40 @@
# mesh-mail-notifier — the default holder of mail-notifier
Built from mesh-mail's source (novox/hq ADR 0378 §9, as amended by ADR 0379, issue 800 part B): it carries a
byte-identical copy of mesh-mail's event shape (`mailevent.go`, pinned by `copies_test.go`, the way
`modules/networkmanager`'s `uplink-tools` carries `ifwatchd`'s own file, ADR 0241). Nothing here names
Betterbird (ADR 0379 §3), and nothing here reaches a mailbox: it only hears what mesh-mail already said.
## What it does
It claims `mail-notifier` (held once per machine, needing `node-notifier` there) and shows each new inbox mail
on this machine's notifier, through `node-notifier.send`, at most once:
- quiet during a first sync or a resync (mesh-mail already says nothing then, `imap.go` ~387)
- quiet for a mail from the account's own address (its address, from `mail-store.accounts`; not known, the
notice is shown)
- quiet for a mail already marked read (`\Seen` in its flags)
- quiet for anything outside the inbox
- of one account's qualifying mail within a sliding minute, the first three are shown one by one; the rest of
that minute's mail is folded into one notice naming the count and the latest sender, rather than shown one
by one
A notice `node-notifier.send` refuses stays pending and is tried again, at most once a minute, without holding
up any other account's notice; nothing is deleted before it is shown. Each notice shown emits this module's
own event `shown` (account and message id, never sender or subject), and is kept in its own state (`shown`) so
a redelivered event is never counted or shown twice — by the mail's message id wherever it has one, so a
UIDVALIDITY reset cannot make an already-shown mail look new.
## Where it runs
**Its own account** (`runs-as`), because it hears mesh-mail's restricted stream `MAIL` (novox/hq ADR 0376 §2):
the machine's runtime runs as the operator's account, which every agent shares and which never hears a
restricted stream. Held once per machine (wherever `mail-notifier` is assigned); the operator grants it the
accounts it announces, in mesh-mail's trusted setting `grants` (`mail-store`'s stream).
## Tools
| tool | does |
|---|---|
| `mail-notifier.status` | every account it is granted, when it last showed each one a notice, and any notice still failing to send (its count, last error, since when). Read-only |
| `mail_notifier_health` | how it is: unhealthy once a notice has failed to show for more than ten minutes, healthy otherwise, even mid-retry. Not (yet) the node-engine's own check — this is one process, judged only by its own tool (ADR 0227 rule 8); a watching process of its own is issue 841, not yet decided. Read-only |
@@ -0,0 +1,31 @@
package main
// mesh-mail-notifier carries a byte-identical copy of mesh-mail's event shape (novox/hq ADR 0378 §9, issue 800
// part B), the way modules/networkmanager's uplink-tools carries ifwatchd's own file (ADR 0241, copies_test.go):
// each builds alone, so each holds its own copy, and this test holds them to one text.
import (
"bytes"
"os"
"path/filepath"
"testing"
)
// source is where mesh-mail's own copy of the file lives.
var source = filepath.Join("..", "..", "..", "mesh-mail", "cmd", "mesh-mail")
func TestCarriesTheSameMailEventAsMeshMail(t *testing.T) {
for _, f := range []string{"mailevent.go"} {
mine, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
theirs, err := os.ReadFile(filepath.Join(source, f))
if err != nil {
t.Fatalf("mesh-mail's copy of %s could not be read: %v", f, err)
}
if !bytes.Equal(mine, theirs) {
t.Errorf("this copy of %s differs from mesh-mail's: change both together", f)
}
}
}
@@ -0,0 +1,62 @@
package main
// mesh-mail-notifier's own health (review round 1 of issue 800, which added this tool beside mail-notifier's
// seat verb status, as mesh-mail's own health tool is mail_health beside its seat tools). A notice that fails
// to send stays pending and is retried, at most once a minute (Flush, notifier.go); the account's
// failing-since stays in the Store, so this tool — its own process, asked separately by the node-engine or by
// mesh_call — sees the same account the live notifier does, with no state of its own to lose between asks.
//
// Unhealthy once an account's notice has failed to show for more than UnhealthyAfter; healthy otherwise, even
// while a notice is mid-retry.
//
// STOP (review round 1 of issue 800, not decided by its note): this tool is served and tested, but module.json
// does not name it as the notify process's own health check — mesh-mail-notifier has one process (notify), and
// the catalogue's gate refuses a module judged only by its own tool, with no check of another kind beside it
// (ADR 0227 rule 8, ADR 0240 rule 2; internal/catalogue/health.go ~297 in mesh-controller). mesh-mail avoids
// this because it has two processes — keep (kind unit) beside watch (kind tool, mail_health); mesh-mail-notifier
// has only notify, so wiring this tool the same way trips the gate. notify's health stays kind unit (liveness,
// as it was before this round) until the lead decides how a one-process module gets an automatic, node-engine
// enforced health of this kind: a second process judged some other way, or a change to the gate's rule.
import (
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// HealthTool is the module's own health tool the node-engine asks (beside mail-notifier.status, the seat's own verb).
const HealthTool = "mail_notifier_health"
// UnhealthyAfter is how long a notice may fail to send before that is unhealthy, not merely retrying.
const UnhealthyAfter = 10 * time.Minute
// Judge is mesh-mail-notifier's health, from the failures the Store keeps.
func Judge(failures map[string]FailureStatus, now time.Time) health.Answer {
if len(failures) == 0 {
return health.Healthy("every notice shown; none pending a retry")
}
accounts := make([]string, 0, len(failures))
for account := range failures {
accounts = append(accounts, account)
}
sort.Strings(accounts)
var problems, retrying []string
for _, account := range accounts {
f := failures[account]
age := now.Sub(f.Since).Round(time.Second)
if now.Sub(f.Since) > UnhealthyAfter {
problems = append(problems, fmt.Sprintf("%s: its notice has failed to show for %s (%d attempt(s), last: %s)",
account, age, f.Count, f.LastError))
} else {
retrying = append(retrying, fmt.Sprintf("%s has been retrying for %s", account, age))
}
}
if len(problems) > 0 {
return health.Unhealthy(strings.Join(problems, "; "))
}
return health.Healthy(strings.Join(retrying, "; "))
}
@@ -0,0 +1,39 @@
package main
import (
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
func TestJudgeIsUnhealthyOnlyPastUnhealthyAfter(t *testing.T) {
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
if a := Judge(nil, now); a.State != health.StateHealthy {
t.Fatalf("no failures: %+v, want healthy", a)
}
recent := map[string]FailureStatus{"work": {Count: 1, LastError: "no answer", Since: now.Add(-2 * time.Minute)}}
if a := Judge(recent, now); a.State != health.StateHealthy {
t.Fatalf("a failure 2m old: %+v, want healthy (still retrying)", a)
}
stale := map[string]FailureStatus{"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)}}
a := Judge(stale, now)
if a.State != health.StateUnhealthy {
t.Fatalf("a failure 11m old: %+v, want unhealthy (past UnhealthyAfter)", a)
}
if !strings.Contains(a.Why, "work") || !strings.Contains(a.Why, "no answer") {
t.Errorf("Why = %q, want it to name the account and the last error", a.Why)
}
mixed := map[string]FailureStatus{
"work": {Count: 11, LastError: "no answer", Since: now.Add(-11 * time.Minute)},
"home": {Count: 1, LastError: "no answer", Since: now.Add(-time.Minute)},
}
if a := Judge(mixed, now); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "work") {
t.Errorf("one stale failure beside one fresh one: %+v, want unhealthy naming work", a)
}
}
@@ -0,0 +1,13 @@
package main
// The shape of a mail-store message event (novox/hq ADR 0378 §2, issue 800 part A), moved here on its own so
// mesh-mail-notifier (issue 800 part B) carries a byte-identical copy, the way modules/networkmanager's
// uplink-tools carries ifwatchd's own file (novox/hq ADR 0241, copies_test.go). Nothing here reaches a mailbox or
// a server; it only says where an account's change is told on the bus and what the three changes are called.
// EventKey is where an account's change is said: mail.<account>.message.<added|changed|removed> (research 049
// document 02 §5), the account by its name in the setting, never its address.
func EventKey(account, change string) string { return "mail." + account + ".message." + change }
// Changes are the three things said of a message.
var Changes = []string{"added", "changed", "removed"}
@@ -0,0 +1,232 @@
// mesh-mail-notifier: the default claimant of mail-notifier, built from mesh-mail's own event shape (novox/hq
// ADR 0378 §9, issue 800 part B). A Go bundle the node's runtime launches, its own account (runs-as, ADR 0376
// §2): the one process serves mail-notifier.status and the module's own health tool (mail_notifier_health,
// review round 1 of issue 800), and, beside stdio.Serve, hears mesh-mail's added events and shows them
// (exactly as modules/mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve) — not
// a separate `notify` process, which systemd gave an empty stdin and so exited at once, every five seconds
// (novox/hq issue 859). What this module says, it says on stderr; stdout is the MCP channel.
package main
import (
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
// AddedPattern is the mesh-mail stream's event this module consumes (its own `consumes` entry, novox/hq ADR
// 0363, ADR 0378 §9): a new, changed or removed message's `added` only.
const AddedPattern = "mesh-mail.mail.*.message.added"
// StatusTool is the mail-notifier seat's one verb this module serves.
const StatusTool = "mail-notifier.status"
// ShownEvent is this module's own event (its `emits`), one per notice shown.
const ShownEvent = "shown"
func main() {
n := &Notifier{
Send: func(args map[string]any) error {
_, err := stdio.Ask(sendKey(), args)
return err
},
Accounts: readAccounts,
Store: busStore{},
EmitShown: func(account, messageID string) error {
return stdio.Emit(ShownEvent, map[string]any{"account": account, "message-id": messageID})
},
Logf: logf,
}
n.init()
tools := []stdio.Tool{
{Name: StatusTool,
Description: "The accounts mesh-mail-notifier announces on this machine, and when it last showed a " +
"notice of new mail (novox/hq ADR 0378 §7). Read-only.",
Run: func(map[string]any) (any, error) { return n.Status() }},
{Name: HealthTool,
Description: "How mesh-mail-notifier is: unhealthy once a notice has failed to show for more than " +
"ten minutes, healthy otherwise, even mid-retry (review round 1 of issue 800). Read-only.",
Run: func(map[string]any) (any, error) {
failures, err := busStore{}.Failures()
if err != nil {
return nil, err
}
return Judge(failures, time.Now()), nil
}},
}
if err := run(n, watch, os.Stdin, os.Stdout, tools); err != nil {
logf("%v", err)
os.Exit(1)
}
}
// sendKey is the ask key Send uses: this machine's own node-notifier, named exactly as desk-channel asks
// another machine's ("seat:node-notifier.send@"+machine) and as the SDK's own Emit reads this machine's name
// (MESH_NODE, which the runtime sets) — not the bare seat, which any machine's node-notifier may answer
// first, leaving this one's notices undelivered (novox/hq issue 865).
func sendKey() string { return "seat:node-notifier.send@" + os.Getenv("MESH_NODE") }
// run is main's startup path, factored so a test can run it: the watcher started before stdio.Serve, exactly
// as mesh-knowledge's serve starts go hear(k) and go background(k) before stdio.Serve — so the watcher keeps
// running whatever stdio.Serve does with the stream it is given, including returning at once when stdin is
// already at EOF, as systemd gave the old separate `notify` process (novox/hq issue 859). The node's runtime
// launches this one process and keeps it alive.
func run(n *Notifier, watcher func(*Notifier), in io.Reader, out io.Writer, tools []stdio.Tool) error {
go watcher(n)
return stdio.ServeOn("", tools, in, out)
}
// watch hears mesh-mail's added events and, once a second, flushes what Notifier.Flush finds ready: an
// account's first mails of its window at once, and its folded overflow once the window is over (ADR 0378 §9,
// the lead's reading, review round 1 of issue 800).
func watch(n *Notifier) { watchWith(n, stdio.Subscribe, time.Sleep) }
// watchWith is watch with its subscribe and sleep factored out, so a test can make a subscribe fail without
// the real bus and without the real wait. A subscribe the runtime refuses is retried, 2 seconds doubling to
// at most a minute, logging each refusal — exactly as modules/mesh-knowledge's hear does — never fatal: a
// refusal here must never take mail-notifier.status and mail_notifier_health down with it (novox/hq issue
// 859, review round 1; the old os.Exit(1) did exactly that).
func watchWith(n *Notifier, subscribe func(string, func(stdio.Envelope) error) error, sleep func(time.Duration)) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
sleep(wait)
err := subscribe(AddedPattern, func(e stdio.Envelope) error {
var body EventBody
if err := json.Unmarshal(e.Body, &body); err != nil {
logf("[mesh-mail-notifier] an added event could not be read; taken and dropped: %v", err)
return nil
}
return n.Add(body)
})
if err == nil {
break
}
logf("[mesh-mail-notifier] does not hear %s yet (%v); asking again in %s", AddedPattern, err, wait)
}
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for range ticker.C {
if err := n.Flush(); err != nil {
logf("[mesh-mail-notifier] a buffered notice could not be shown: %v", err)
}
}
}
// readAccounts asks mail-store.accounts for the accounts this module is granted, by name to own address.
func readAccounts() (map[string]string, error) {
raw, err := stdio.Ask("seat:mail-store.accounts", map[string]any{})
if err != nil {
return nil, err
}
var answer struct {
Accounts []struct {
Account string `json:"account"`
Address string `json:"address"`
} `json:"accounts"`
}
if err := json.Unmarshal(raw, &answer); err != nil {
return nil, err
}
out := map[string]string{}
for _, a := range answer.Accounts {
out[a.Account] = a.Address
}
return out, nil
}
// busStore keeps the notifier's own state (the `shown` bucket the manifest declares): which messages were
// shown already, each account's last notice, and each account's notice still failing to send.
type busStore struct{}
func stateKey(parts ...string) string {
words := make([]string, len(parts))
for i, p := range parts {
var b strings.Builder
for _, r := range p {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-':
b.WriteRune(r)
default:
b.WriteRune('_')
}
}
words[i] = b.String()
}
return strings.Join(words, "_")
}
func (busStore) HandleSeen(handle string) (bool, error) {
e, err := stdio.State("shown").Get("handle_" + stateKey(strings.Split(handle, "\x1f")...))
return e != nil, err
}
func (busStore) MarkHandle(handle string) error {
_, err := stdio.State("shown").Put("handle_"+stateKey(strings.Split(handle, "\x1f")...), true)
return err
}
func (busStore) PutStatus(account string, s AccountStatus) error {
_, err := stdio.State("shown").Put("status_"+stateKey(account), s)
return err
}
func (busStore) Statuses() (map[string]AccountStatus, error) {
keys, err := stdio.State("shown").Keys()
if err != nil {
return nil, err
}
out := map[string]AccountStatus{}
for _, k := range keys {
account, ok := strings.CutPrefix(k, "status_")
if !ok {
continue
}
e, err := stdio.State("shown").Get(k)
if err != nil || e == nil {
continue
}
var s AccountStatus
if json.Unmarshal(e.Value, &s) == nil {
out[account] = s
}
}
return out, nil
}
func (busStore) PutFailure(account string, f FailureStatus) error {
_, err := stdio.State("shown").Put("failure_"+stateKey(account), f)
return err
}
func (busStore) ClearFailure(account string) error {
return stdio.State("shown").Delete("failure_" + stateKey(account))
}
func (busStore) Failures() (map[string]FailureStatus, error) {
keys, err := stdio.State("shown").Keys()
if err != nil {
return nil, err
}
out := map[string]FailureStatus{}
for _, k := range keys {
account, ok := strings.CutPrefix(k, "failure_")
if !ok {
continue
}
e, err := stdio.State("shown").Get(k)
if err != nil || e == nil {
continue
}
var f FailureStatus
if json.Unmarshal(e.Value, &f) == nil {
out[account] = f
}
}
return out, nil
}
@@ -0,0 +1,38 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
// The manifest's own shape (novox/hq issue 859): the watch loop runs in the base invocation beside
// stdio.Serve, so no resource declares a separate `notify` process any more — the thing that exited at once,
// every five seconds, because a systemd-run process's stdin is empty and stdio.Serve returns at EOF.
type manifestResources struct {
Resources []map[string]any `json:"resources"`
}
func readManifestResources(t *testing.T) manifestResources {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m manifestResources
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatalf("module.json: %v", err)
}
return m
}
func TestNoProcessResourceIsDeclared(t *testing.T) {
m := readManifestResources(t)
for _, r := range m.Resources {
if r["type"] == "process" {
t.Fatalf("module.json still declares a process resource (%v): the watcher runs in the base invocation now, not a separate process (novox/hq issue 859)", r)
}
}
}
@@ -0,0 +1,537 @@
package main
// mesh-mail-notifier: the default holder of mail-notifier (novox/hq ADR 0378 §7, as amended by ADR 0379 §4,
// issue 800 part B). It hears mesh-mail's added events on its own account (runs-as, ADR 0376 §2) and shows each
// new inbox mail at most once, through node-notifier.send: quiet during a first sync (mesh-mail already says
// nothing then, imap.go ~387), for mail from the account's own address, for mail already marked read (\Seen),
// and for anything outside the inbox. For each new qualifying mail of an account, arriving at t, N counts that
// account's qualifying mail — shown or folded alike, each counted once — that arrived in the 60 seconds up to
// and including t: N <= FoldOver is shown at once; N > FoldOver is folded. The first folded mail opens the
// account's fold; every folded mail joins it; the fold is shown, as one notice naming how many it holds and
// the latest sender, once Window has passed since its own first mail — after which a new fold may open. The
// count is a true sliding window of arrivals, not a per-session quota that only resets after a quiet Window:
// review round 2's reading kept a session's count alive, and so folding, for as long as any gap stayed under
// Window, which folds a steady trickle (one mail every 50s, say) forever even though no burst ever happened;
// review round 3 of issue 800 replaces that reading with this one. Nothing pending — a folded mail awaiting
// its fold's deadline, or a failed send awaiting its retry — is ever dropped (novox/hq ADR 0378 §9, review
// round 3 of issue 800, which replaces the round 1 and round 2 readings).
//
// Nothing here names Betterbird (ADR 0379 §3).
import (
"sort"
"strconv"
"strings"
"sync"
"time"
)
// Inbox is the one folder a new mail is shown for.
const Inbox = "INBOX"
// Seen is the flag that means a mail was already read: shown for nothing it marks.
const SeenFlag = "\\Seen"
// FoldOver is how many new mails from one account, within one window, are still shown one by one; the rest of
// that window is folded into a single notice (novox/hq ADR 0378 §9: "more than three").
const FoldOver = 3
// Window is the trailing 60 seconds ADR 0378 §9 counts a mail's N over (every qualifying mail of the account,
// shown or folded alike, that arrived in the 60 seconds up to and including this one), and the time a fold
// waits, from its own first mail, before it is shown (review round 3 of issue 800, which replaces the round 1
// and round 2 readings: the count is a true sliding window of arrivals, not a per-session quota that stays
// alive, and so keeps folding, for as long as any gap between mails stays under Window).
const Window = time.Minute
// RetryBackoff is how long the notifier waits, after an account's notice fails to send, before it tries that
// account again: at most once a minute (review round 1 of issue 800), so a send that keeps failing is not
// retried on every Flush.
const RetryBackoff = time.Minute
// AccountsCacheFor is how long mail-store.accounts's answer is reused before it is asked again (review round
// 1 of issue 800, bullet 5): neither the quiet rule nor status asks it more than once a minute.
const AccountsCacheFor = time.Minute
// Person is a sender or a recipient, as mesh-mail's event says it.
type Person struct {
Name string `json:"name,omitempty"`
Address string `json:"address"`
}
// EventBody is what mesh-mail's added event says of a message (metadata only; mailevent.go names where it is
// said). Unknown fields are ignored: the notifier reads only what it needs.
type EventBody struct {
Account string `json:"account"`
Folder string `json:"folder"`
UID uint32 `json:"uid"`
UIDValidity uint32 `json:"uidvalidity"`
MessageID string `json:"message-id"`
Subject string `json:"subject"`
From []Person `json:"from"`
Flags []string `json:"flags"`
}
// Handle names one message, stably, for the dedupe state (novox/hq ADR 0378 §9: "a redelivered event is not
// shown twice"). The message id is the key wherever a message carries one, so a UIDVALIDITY reset — which
// changes every UID in a folder — can never make an already-shown mail look new, or a genuinely new one look
// already shown; only a message with no message id falls back to its folder, UIDVALIDITY and UID (review
// round 1 of issue 800).
func (e EventBody) Handle() string {
if e.MessageID != "" {
return e.Account + "\x1f" + e.MessageID
}
return e.Account + "\x1f" + e.Folder + "\x1f" +
strconv.FormatUint(uint64(e.UIDValidity), 10) + "\x1f" + strconv.FormatUint(uint64(e.UID), 10)
}
func (e EventBody) hasFlag(flag string) bool {
for _, f := range e.Flags {
if strings.EqualFold(f, flag) {
return true
}
}
return false
}
// sender is how a message's first sender reads in a notice: its name and address, or its address alone.
func (e EventBody) sender() string {
if len(e.From) == 0 {
return "(unknown sender)"
}
p := e.From[0]
if p.Name == "" {
return p.Address
}
return p.Name + " <" + p.Address + ">"
}
func (e EventBody) fromAddress() string {
if len(e.From) == 0 {
return ""
}
return e.From[0].Address
}
// Notice is one notification the notifier shows: its args for node-notifier.send, and the message its `shown`
// event names (the latest one, for a folded notice).
type Notice struct {
Account string
Args map[string]any
Handles []string // every message this notice accounts for, marked seen once shown
Latest EventBody
}
// FailureStatus is what the notifier knows of an account's notice still failing to send (review round 1 of
// issue 800): how many attempts, the last error, and since when. Kept in the Store, not only in memory, so
// the health tool — its own process, asked separately by the node-engine — sees the same account the live
// notifier does.
type FailureStatus struct {
Count int `json:"count"`
LastError string `json:"last-error"`
Since time.Time `json:"since"`
}
// Store is what the notifier keeps of its own (the `shown` state the manifest declares): whether a message
// was shown already, each account's last notice, and each account's notice still failing to send.
type Store interface {
HandleSeen(handle string) (bool, error)
MarkHandle(handle string) error
PutStatus(account string, s AccountStatus) error
Statuses() (map[string]AccountStatus, error)
PutFailure(account string, f FailureStatus) error
ClearFailure(account string) error
Failures() (map[string]FailureStatus, error)
}
// AccountStatus is what the notifier last showed of one account.
type AccountStatus struct {
At time.Time `json:"at"`
Summary string `json:"summary"`
}
// accountBuffer is one account's state: history is every qualifying mail's arrival time, kept only for the
// trailing Window, that N is counted against (review round 3 of issue 800); immediate holds mail waiting to be
// shown one by one at the next Flush; folded holds the account's current fold (its first FoldOver mails within
// Window are never folded, so a fold only ever holds the overflow from the fourth qualifying mail of its own
// window onward), due as one notice once Window has passed since foldStart, its first mail's arrival.
type accountBuffer struct {
history []time.Time
foldStart time.Time // when the current fold's first mail arrived; zero while nothing is folded
immediate []EventBody
folded []EventBody
}
// Notifier buffers qualifying mail per account and shows it: the first FoldOver mails of a window one by one,
// the rest folded once the window ends. A notice that fails to send is tried again, at most once a minute,
// without holding up any other account's notice.
type Notifier struct {
// Send calls node-notifier.send on this machine.
Send func(args map[string]any) error
// Accounts reads the accounts the notifier is granted, by mail-store.accounts: each one's own address, for
// the quiet rule (novox/hq ADR 0378 §9's LEAD note). An account not known there is shown anyway.
Accounts func() (map[string]string, error)
Store Store
// EmitShown says mesh-mail-notifier's own event `shown`: account and message id, never sender or subject.
EmitShown func(account, messageID string) error
Now func() time.Time
Logf func(string, ...any)
mu sync.Mutex
pending map[string]*accountBuffer
buffered map[string]bool // a message's Handle while it is waiting to be shown: never counted twice
cooldown map[string]time.Time // account -> not retried before this time, after a failed send
accountsAt time.Time
accountsVal map[string]string
accountsErr error
}
func (n *Notifier) init() {
n.mu.Lock()
defer n.mu.Unlock()
if n.pending == nil {
n.pending = map[string]*accountBuffer{}
n.buffered = map[string]bool{}
n.cooldown = map[string]time.Time{}
}
if n.Now == nil {
n.Now = time.Now
}
if n.Logf == nil {
n.Logf = func(string, ...any) {}
}
}
// Add takes one added event. It returns an error only to have the event offered again (the handler
// Subscribe wraps this in follows that rule).
func (n *Notifier) Add(e EventBody) error {
n.init()
handle := e.Handle()
seen, err := n.Store.HandleSeen(handle)
if err != nil {
return err
}
if seen {
return nil // a redelivery of a message already shown (or already judged not to be)
}
own, known, err := n.ownAddress(e.Account)
if err != nil {
return err
}
if !n.qualifies(e, own, known) {
return nil
}
n.mu.Lock()
defer n.mu.Unlock()
if n.buffered[handle] {
return nil // a redelivery of a message still waiting to be shown: never counted twice
}
now := n.Now()
b := n.pending[e.Account]
if b == nil {
b = &accountBuffer{}
n.pending[e.Account] = b
}
// N is every qualifying mail of this account — shown or folded alike, each counted once — that arrived in
// the 60 seconds up to and including now: a true sliding window over arrivals, not a per-session quota that
// stays alive, and so keeps folding, for as long as any gap stays under Window (review round 3 of issue
// 800, which replaces the round 1 and round 2 readings; nothing in this count depends on whether an earlier
// mail has been shown yet).
cutoff := now.Add(-Window)
history := make([]time.Time, 0, len(b.history)+1)
for _, at := range b.history {
if !at.Before(cutoff) {
history = append(history, at)
}
}
b.history = append(history, now)
n.buffered[handle] = true
if len(b.history) <= FoldOver {
b.immediate = append(b.immediate, e)
} else {
if len(b.folded) == 0 {
b.foldStart = now
}
b.folded = append(b.folded, e)
}
return nil
}
// qualifies says whether a mail is shown at all: a new inbox mail, not the account's own, not already read.
func (n *Notifier) qualifies(e EventBody, ownAddress string, ownKnown bool) bool {
if !strings.EqualFold(e.Folder, Inbox) {
return false
}
if e.hasFlag(SeenFlag) {
return false
}
if ownKnown && ownAddress != "" && strings.EqualFold(e.fromAddress(), ownAddress) {
return false
}
return true
}
func (n *Notifier) ownAddress(account string) (string, bool, error) {
accounts, err := n.readAccounts()
if err != nil {
// Not known: ADR 0378 §9's LEAD note says the notice is shown when the address cannot be read. Logged,
// not retried: mail-store being unreachable is not this event's fault.
n.Logf("[mesh-mail-notifier] the account's own address could not be read (%v); the notice is shown", err)
return "", false, nil
}
addr, ok := accounts[account]
return addr, ok, nil
}
// readAccounts asks mail-store.accounts, reusing its last answer for AccountsCacheFor so neither the quiet
// rule nor status reads it more than once a minute (review round 1 of issue 800, bullet 5).
func (n *Notifier) readAccounts() (map[string]string, error) {
n.mu.Lock()
if n.accountsVal != nil && n.Now().Sub(n.accountsAt) < AccountsCacheFor {
val, err := n.accountsVal, n.accountsErr
n.mu.Unlock()
return val, err
}
n.mu.Unlock()
val, err := n.Accounts()
n.mu.Lock()
n.accountsVal, n.accountsErr, n.accountsAt = val, err, n.Now()
n.mu.Unlock()
return val, err
}
// Flush shows what is ready: each account's first FoldOver mails of its window, one by one, as soon as Flush
// runs, and the window's folded overflow once the window has run its minute. Accounts are judged in a sorted
// order, not a map's iteration order, so the tests do not depend on it (review round 1 of issue 800, BLOCKING
// 1). A notice that fails to send leaves its mail pending, tried again once RetryBackoff has passed, and never
// holds up another account's notice (BLOCKING 2): nothing is deleted before it is shown.
func (n *Notifier) Flush() error {
n.init()
now := n.Now()
n.mu.Lock()
accounts := make([]string, 0, len(n.pending))
for account := range n.pending {
accounts = append(accounts, account)
}
n.mu.Unlock()
sort.Strings(accounts)
for _, account := range accounts {
n.flushAccount(account, now)
}
return nil
}
// flushAccount ships one account's ready notices (its immediate mail, and its fold once Window has passed
// since the fold's own first mail), and keeps whatever fails to send pending, under cooldown, for the next
// Flush. Nothing is deleted before it is shown (review round 1 BLOCKING 2), and nothing pending is ever lost
// (review round 2 BLOCKING, kept under review round 3's rule; see Add).
func (n *Notifier) flushAccount(account string, now time.Time) {
n.mu.Lock()
if cd, ok := n.cooldown[account]; ok && now.Before(cd) {
n.mu.Unlock()
return // backing off after a failed send; tried again once RetryBackoff has passed
}
b := n.pending[account]
if b == nil {
n.mu.Unlock()
return
}
immediate := append([]EventBody(nil), b.immediate...)
// The fold is due Window after its own first mail (review round 3 of issue 800).
foldDue := len(b.folded) > 0 && now.Sub(b.foldStart) >= Window
var fold []EventBody
if foldDue {
fold = append([]EventBody(nil), b.folded...)
}
n.mu.Unlock()
if len(immediate) == 0 && len(fold) == 0 {
return
}
sentImmediate := 0
var failErr error
for _, e := range immediate {
if err := n.show(n.singleNotice(account, e)); err != nil {
failErr = err
break
}
sentImmediate++
}
foldSent := false
if failErr == nil && len(fold) > 0 {
if err := n.show(n.foldNotice(account, fold, len(fold))); err != nil {
failErr = err
} else {
foldSent = true
}
}
n.mu.Lock()
if cur := n.pending[account]; cur == b {
cur.immediate = cur.immediate[sentImmediate:]
if foldSent {
cur.folded = cur.folded[len(fold):]
if len(cur.folded) == 0 {
cur.foldStart = time.Time{} // nothing left folded: a new fold may open (review round 3 of issue 800)
} else {
// More mail folded in while this batch was being sent: its own Window starts now, counted from
// when it was first noticed. Nothing of it is lost either way.
cur.foldStart = now
}
}
}
if failErr != nil {
n.cooldown[account] = now.Add(RetryBackoff)
n.recordFailureLocked(account, failErr, now)
} else {
delete(n.cooldown, account)
n.clearFailureLocked(account)
}
n.mu.Unlock()
}
// recordFailureLocked keeps an account's failure in the Store (mu held by the caller, released around the
// Store calls since they may reach the bus): its count, its last error, and since when, so a later Flush and
// the health tool agree on how long it has been failing.
func (n *Notifier) recordFailureLocked(account string, sendErr error, now time.Time) {
n.mu.Unlock()
defer n.mu.Lock()
since := now
count := 1
if prev, err := n.Store.Failures(); err == nil {
if p, ok := prev[account]; ok {
since = p.Since
count = p.Count + 1
}
}
if err := n.Store.PutFailure(account, FailureStatus{Count: count, LastError: sendErr.Error(), Since: since}); err != nil {
n.Logf("[mesh-mail-notifier] a failure could not be kept (%v): the health tool may not see it", err)
}
}
func (n *Notifier) clearFailureLocked(account string) {
n.mu.Unlock()
defer n.mu.Lock()
if err := n.Store.ClearFailure(account); err != nil {
n.Logf("[mesh-mail-notifier] a cleared failure could not be kept cleared: %v", err)
}
}
// singleNotice is one mail shown on its own: one of an account's first FoldOver mails of its window.
func (n *Notifier) singleNotice(account string, e EventBody) Notice {
return Notice{
Account: account,
Args: map[string]any{
"summary": e.Subject,
"body": e.sender() + " — " + account,
"app_name": "mesh-mail",
},
Handles: []string{e.Handle()},
Latest: e,
}
}
// foldNotice is one notice for everything the account's current fold holds — its qualifying mail from the
// fourth within some trailing 60 seconds onward — naming how many mail that fold holds (not the account's
// whole history of arrivals) and the latest sender (novox/hq ADR 0378 §9, review round 3 of issue 800, which
// replaces the round 1 and round 2 readings: "<k> more new mail(s) in <account>, latest from <sender of the
// last>", "mail" for one and "mails" for more).
func (n *Notifier) foldNotice(account string, events []EventBody, count int) Notice {
latest := events[len(events)-1]
notice := Notice{
Account: account,
Args: map[string]any{
"summary": strconv.Itoa(count) + " more new " + mailWord(count) + " in " + account,
"body": "latest from " + latest.sender(),
"app_name": "mesh-mail",
},
Latest: latest,
}
for _, e := range events {
notice.Handles = append(notice.Handles, e.Handle())
}
return notice
}
// mailWord says "mail" for a count of one, "mails" for anything else (review round 2 of issue 800, bullet 3;
// kept by review round 3).
func mailWord(count int) string {
if count == 1 {
return "mail"
}
return "mails"
}
// show sends one notice, marks every message it accounts for seen, keeps the account's status, and emits
// `shown`. Nothing is marked, kept or emitted when Send fails: the caller keeps the notice pending instead.
func (n *Notifier) show(notice Notice) error {
if err := n.Send(notice.Args); err != nil {
n.Logf("[mesh-mail-notifier] %s's notice could not be shown: %v", notice.Account, err)
return err
}
n.mu.Lock()
for _, h := range notice.Handles {
delete(n.buffered, h)
}
n.mu.Unlock()
for _, h := range notice.Handles {
if err := n.Store.MarkHandle(h); err != nil {
n.Logf("[mesh-mail-notifier] shown, but not kept as shown (%v): a redelivery may show it again", err)
}
}
summary, _ := notice.Args["summary"].(string)
if err := n.Store.PutStatus(notice.Account, AccountStatus{At: n.Now(), Summary: summary}); err != nil {
n.Logf("[mesh-mail-notifier] shown, but the status could not be kept: %v", err)
}
return n.EmitShown(notice.Account, notice.Latest.MessageID)
}
// Status is the notifier's account of itself, for mail-notifier.status (novox/hq ADR 0378 §7: "accounts it
// announces there, when it last showed a notice"; review round 1 of issue 800: the accounts
// mail-store.accounts gives it, and the failures it is retrying).
func (n *Notifier) Status() (map[string]any, error) {
statuses, err := n.Store.Statuses()
if err != nil {
return nil, err
}
failures, err := n.Store.Failures()
if err != nil {
return nil, err
}
known, knownErr := n.readAccounts()
accounts := map[string]any{}
for account := range known {
accounts[account] = map[string]any{}
}
for account, s := range statuses {
accounts[account] = map[string]any{"at": s.At.UTC().Format(time.RFC3339), "shown": s.Summary}
}
out := map[string]any{
"accounts": accounts,
"rules": []string{
"it shows each new inbox mail at most once, through node-notifier.send on this machine (novox/hq ADR 0378 §7)",
"quiet during a first sync, for mail from the account's own address, for mail already marked read, and for anything outside the inbox",
"of one account's qualifying mail, one arriving when fewer than three of the account's mails fell in the 60 seconds before it is shown at once; the rest is folded and shown, as one notice naming how many it holds and the latest sender, 60 seconds after the fold's own first mail",
},
}
if knownErr != nil {
out["accounts-error"] = knownErr.Error()
}
if len(failures) > 0 {
failing := map[string]any{}
for account, f := range failures {
failing[account] = map[string]any{
"count": f.Count,
"last-error": f.LastError,
"since": f.Since.UTC().Format(time.RFC3339),
}
}
out["failing"] = failing
}
return out, nil
}
@@ -0,0 +1,569 @@
package main
import (
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
)
// fakeStore is Store kept in memory, for the test alone.
type fakeStore struct {
mu sync.Mutex
handles map[string]bool
statuses map[string]AccountStatus
failures map[string]FailureStatus
}
func newFakeStore() *fakeStore {
return &fakeStore{
handles: map[string]bool{},
statuses: map[string]AccountStatus{},
failures: map[string]FailureStatus{},
}
}
func (s *fakeStore) HandleSeen(handle string) (bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
return s.handles[handle], nil
}
func (s *fakeStore) MarkHandle(handle string) error {
s.mu.Lock()
defer s.mu.Unlock()
s.handles[handle] = true
return nil
}
func (s *fakeStore) PutStatus(account string, st AccountStatus) error {
s.mu.Lock()
defer s.mu.Unlock()
s.statuses[account] = st
return nil
}
func (s *fakeStore) Statuses() (map[string]AccountStatus, error) {
s.mu.Lock()
defer s.mu.Unlock()
out := map[string]AccountStatus{}
for k, v := range s.statuses {
out[k] = v
}
return out, nil
}
func (s *fakeStore) PutFailure(account string, f FailureStatus) error {
s.mu.Lock()
defer s.mu.Unlock()
s.failures[account] = f
return nil
}
func (s *fakeStore) ClearFailure(account string) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.failures, account)
return nil
}
func (s *fakeStore) Failures() (map[string]FailureStatus, error) {
s.mu.Lock()
defer s.mu.Unlock()
out := map[string]FailureStatus{}
for k, v := range s.failures {
out[k] = v
}
return out, nil
}
// fakeClock is a clock the test moves by hand, so Flush's window needs no real sleep.
type fakeClock struct {
mu sync.Mutex
now time.Time
}
func (c *fakeClock) Now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.now
}
func (c *fakeClock) Advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.now = c.now.Add(d)
}
func newTestNotifier(t *testing.T, clock *fakeClock, store *fakeStore, sent *[]map[string]any) *Notifier {
t.Helper()
return &Notifier{
Now: clock.Now,
Send: func(args map[string]any) error {
*sent = append(*sent, args)
return nil
},
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
Store: store,
EmitShown: func(string, string) error { return nil },
Logf: t.Logf,
}
}
func mail(account string, uid int, messageID, subject, senderName string) EventBody {
return EventBody{Account: account, Folder: "INBOX", UID: uint32(uid), MessageID: messageID, Subject: subject,
From: []Person{{Name: senderName, Address: "s@example.com"}}}
}
// TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder is the review's first test for BLOCKING 1: Flush
// used to walk a map, so which account's notice went out first was whichever order Go's map iteration gave
// that run. Run with -count=50: green every time only once Flush sorts the accounts itself.
func TestFlushShowsAccountsInASortedOrderNotAMapsIterationOrder(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
for _, account := range []string{"zeta", "mango", "alpha", "delta"} {
if err := n.Add(mail(account, 1, account+"-m1", account, "Someone")); err != nil {
t.Fatalf("Add(%s): %v", account, err)
}
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
want := []string{"alpha", "delta", "mango", "zeta"}
if len(sent) != len(want) {
t.Fatalf("sent %d notice(s), want %d: %+v", len(sent), len(want), sent)
}
for i, account := range want {
if sent[i]["summary"] != account {
t.Fatalf("sent[%d]'s summary is %v, want %q: accounts must be flushed in sorted order: %+v",
i, sent[i]["summary"], account, sent)
}
}
}
// TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts is the review's second test for BLOCKING 2:
// a notice node-notifier.send refuses stays pending (nothing is deleted before it is shown), is tried again
// at the next Flush no sooner than RetryBackoff, and another account's notice still goes out meanwhile.
func TestAFailedSendIsRetriedWithoutLosingItOrHoldingUpOtherAccounts(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
failSend := true
n := &Notifier{
Now: clock.Now,
Send: func(args map[string]any) error {
if args["summary"] == "Broken" && failSend {
return errors.New("node-notifier.send: no answer")
}
sent = append(sent, args)
return nil
},
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
Store: store,
EmitShown: func(string, string) error { return nil },
Logf: t.Logf,
}
if err := n.Add(mail("broke", 1, "b1", "Broken", "Someone")); err != nil {
t.Fatalf("Add(broke): %v", err)
}
if err := n.Add(mail("ok", 1, "o1", "Fine", "Someone")); err != nil {
t.Fatalf("Add(ok): %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 || sent[0]["summary"] != "Fine" {
t.Fatalf("sent = %+v, want only ok's notice: broke's failure must not hold it up", sent)
}
if failures, err := store.Failures(); err != nil || failures["broke"].Count != 1 {
t.Fatalf("Store.Failures() = %+v, %v; want broke recorded once", failures, err)
}
// Before RetryBackoff has passed, broke's notice is not retried yet.
clock.Advance(30 * time.Second)
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 {
t.Fatalf("sent %d notice(s) before RetryBackoff elapsed, want 1 (broke not retried yet): %+v", len(sent), sent)
}
// Once RetryBackoff has passed, the next Flush tries it again; nothing was lost.
failSend = false
clock.Advance(RetryBackoff)
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 2 || sent[1]["summary"] != "Broken" {
t.Fatalf("sent = %+v, want broke's notice too, after RetryBackoff", sent)
}
if failures, _ := store.Failures(); len(failures) != 0 {
t.Errorf("Store.Failures() = %+v, want it cleared once broke's notice was shown", failures)
}
}
// TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd is the review's third test (BLOCKING 3,
// ADR 0378 §9, the lead's reading, the fold notice's wording updated by review round 3 of issue 800): the
// first three of an account's qualifying mail within a minute are shown one by one as Flush runs; the fourth
// onward is folded and shown as one notice once the window — a minute from the first of them — is over.
func TestABurstShowsTheFirstThreeAtOnceAndFoldsTheRestAtTheWindowsEnd(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
for i := 1; i <= 4; i++ {
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
t.Fatalf("Add mail %d: %v", i, err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
clock.Advance(10 * time.Second)
}
if len(sent) != 3 {
t.Fatalf("4 mails 10s apart: sent %d notice(s) before the window ends, want 3 shown one by one: %+v", len(sent), sent)
}
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3"} {
if sent[i]["summary"] != want {
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
}
}
// The window is over a minute after it opened (the first mail): the fourth, folded, is shown now.
clock.Advance(Window)
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 4 {
t.Fatalf("after the window ended, sent %d notice(s), want 4 (the folded fourth mail too): %+v", len(sent), sent)
}
if sent[3]["summary"] != "1 more new mail in home" || !strings.Contains(sent[3]["body"].(string), "Sender 4") {
t.Errorf("the fold notice is %+v, want it to count only what the fold holds (1), naming Sender 4", sent[3])
}
}
// TestARedeliveredEventIsNeverShownOrCountedTwice is the review's fourth test: a redelivery of a mail already
// shown is dropped (the Store knows it), and a redelivery of a mail still buffered, waiting to be folded, is
// never counted a second time toward the fold.
func TestARedeliveredEventIsNeverShownOrCountedTwice(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
one := mail(account, 1, "m1", "Hi", "Alice")
if err := n.Add(one); err != nil {
t.Fatalf("Add: %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 {
t.Fatalf("sent %d notice(s), want 1", len(sent))
}
if err := n.Add(one); err != nil { // redelivered after it was shown
t.Fatalf("Add (redelivered, already shown): %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 {
t.Fatalf("a redelivered, already-shown event was shown again: sent now %+v", sent)
}
for i := 2; i <= 4; i++ {
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), "Hi", fmt.Sprintf("Sender %d", i))); err != nil {
t.Fatalf("Add mail %d: %v", i, err)
}
}
fourth := mail(account, 4, "m4", "Hi", "Sender 4")
if err := n.Add(fourth); err != nil { // redelivered while still buffered, folded, not yet shown
t.Fatalf("Add (redelivered, buffered): %v", err)
}
clock.Advance(Window)
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 4 {
t.Fatalf("sent %d notice(s), want 4 (one, mail 2, mail 3, and one fold notice): %+v", len(sent), sent)
}
if sent[3]["summary"] != "1 more new mail in home" {
t.Errorf("the fold notice is %+v, want it to count the redelivered fourth mail once, not twice, among the fold's 1", sent[3])
}
}
// TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange is the review's fifth test: the dedupe key is the
// account and the message id wherever one is given, so a UIDVALIDITY reset — which changes every UID in a
// folder — cannot make an already-shown mail look new. Only a message with no message id falls back to its
// folder, UIDVALIDITY and UID, and that fallback still tells two different mails apart.
func TestDedupeKeyIsTheMessageIDAcrossAUIDVALIDITYChange(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
first := EventBody{Account: account, Folder: "INBOX", UID: 5, UIDValidity: 1, MessageID: "stable-id",
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
if err := n.Add(first); err != nil {
t.Fatalf("Add: %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 {
t.Fatalf("sent %d, want 1", len(sent))
}
// The same message, redelivered after a UIDVALIDITY reset changed its UID: same message id, not shown again.
resent := EventBody{Account: account, Folder: "INBOX", UID: 99, UIDValidity: 2, MessageID: "stable-id",
Subject: "Hi", From: []Person{{Address: "a@example.com"}}}
if err := n.Add(resent); err != nil {
t.Fatalf("Add (UIDVALIDITY reset, same message id): %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 1 {
t.Fatalf("a mail with the same message id after a UIDVALIDITY reset was shown again: sent now %+v", sent)
}
// Two different mails with no message id, at the same UID but a different UIDVALIDITY, are not confused.
noID1 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 1, Subject: "No id",
From: []Person{{Address: "a@example.com"}}}
noID2 := EventBody{Account: account, Folder: "INBOX", UID: 7, UIDValidity: 2, Subject: "No id, new validity",
From: []Person{{Address: "a@example.com"}}}
if err := n.Add(noID1); err != nil {
t.Fatalf("Add noID1: %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if err := n.Add(noID2); err != nil {
t.Fatalf("Add noID2: %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 3 {
t.Fatalf("sent %d, want 3: noID1 and noID2 are different mails, their UIDVALIDITY differs: %+v", len(sent), sent)
}
}
// TestQualifyingRulesStillHold keeps the original qualifying rules proven (an already-read mail, a mail from
// the account's own address, and a mail outside the inbox are never shown), now that buffering is per window
// rather than per quiet period.
func TestQualifyingRulesStillHold(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := &Notifier{
Now: clock.Now,
Send: func(args map[string]any) error { sent = append(sent, args); return nil },
Accounts: func() (map[string]string, error) { return map[string]string{"work": "me@work.example"}, nil },
Store: store,
EmitShown: func(string, string) error { return nil },
Logf: t.Logf,
}
already := EventBody{Account: "work", Folder: "INBOX", UID: 2, MessageID: "m2", Subject: "Old",
From: []Person{{Address: "bob@example.com"}}, Flags: []string{"\\Seen"}}
own := EventBody{Account: "work", Folder: "INBOX", UID: 3, MessageID: "m3", Subject: "Sent",
From: []Person{{Address: "me@work.example"}}}
elsewhere := EventBody{Account: "work", Folder: "Archive", UID: 4, MessageID: "m4", Subject: "Filed",
From: []Person{{Address: "carol@example.com"}}}
for _, e := range []EventBody{already, own, elsewhere} {
if err := n.Add(e); err != nil {
t.Fatalf("Add(%+v): %v", e, err)
}
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 0 {
t.Fatalf("sent %+v, want nothing: an already-read mail, the account's own mail and mail outside the inbox never qualify", sent)
}
}
// TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail is review round 2 of issue 800's first test (BLOCKING):
// a send that fails keeps its mail pending; once the cooldown has passed, but before the next Flush retries
// it, a new qualifying mail arrives. Neither mail is lost: both are shown once the retry succeeds.
func TestABufferIsNeverReplacedWhileItHoldsAnUnsentMail(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
failSend := true
n := &Notifier{
Now: clock.Now,
Send: func(args map[string]any) error {
if failSend {
return errors.New("node-notifier.send: no answer")
}
sent = append(sent, args)
return nil
},
Accounts: func() (map[string]string, error) { return map[string]string{}, nil },
Store: store,
EmitShown: func(string, string) error { return nil },
Logf: t.Logf,
}
account := "home"
if err := n.Add(mail(account, 1, "m1", "Mail 1", "Sender 1")); err != nil {
t.Fatalf("Add mail 1: %v", err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 0 {
t.Fatalf("sent %d notice(s), want 0: the send failed", len(sent))
}
// The cooldown passes, but no retry has happened yet (Flush is not called again before the new mail
// arrives): a buffer this old must not be thrown away for a fresh one.
clock.Advance(RetryBackoff)
if err := n.Add(mail(account, 2, "m2", "Mail 2", "Sender 2")); err != nil {
t.Fatalf("Add mail 2: %v", err)
}
failSend = false
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 2 {
t.Fatalf("sent %d notice(s), want 2 (mail 1's retry and mail 2, neither lost): %+v", len(sent), sent)
}
for i, want := range []string{"Mail 1", "Mail 2"} {
if sent[i]["summary"] != want {
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
}
}
}
// TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota is review round 3 of issue 800's first test,
// replacing round 2's (which contradicted round 3's rule): N, for a mail arriving at t, is every qualifying
// mail of the account — shown or folded alike — that arrived in the 60 seconds up to and including t. Mails at
// 0, 50, 55, 70, 75 and 80 seconds: at 70s the preceding 60 seconds (from 10s) hold 50, 55 and 70 themselves
// (0s has aged out), N=3, so 70 is still shown at once, not folded; at 75s the preceding 60 seconds (from 15s)
// still hold 50 too, so it, 55, 70 and 75 themselves make N=4, and 75 folds; 80 folds with it (its own
// preceding 60 seconds hold 50, 55, 70, 75 and 80, N=5). The fold is due 60 seconds after its own first mail
// (75s), at 135s, as one notice naming only what it holds (2) and the latest sender (80s's).
func TestTheFoldFollowsATrueSlidingWindowOfArrivalsNotASessionQuota(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
deltas := []time.Duration{0, 50 * time.Second, 5 * time.Second, 15 * time.Second, 5 * time.Second, 5 * time.Second}
for i, d := range deltas {
clock.Advance(d)
if err := n.Add(mail(account, i+1, fmt.Sprintf("m%d", i+1), fmt.Sprintf("Mail %d", i+1), fmt.Sprintf("Sender %d", i+1))); err != nil {
t.Fatalf("Add mail %d: %v", i+1, err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush at mail %d: %v", i+1, err)
}
}
if len(sent) != 4 {
t.Fatalf("sent %d notice(s), want 4 shown one by one (0s, 50s, 55s, 70s): %+v", len(sent), sent)
}
for i, want := range []string{"Mail 1", "Mail 2", "Mail 3", "Mail 4"} {
if sent[i]["summary"] != want {
t.Errorf("sent[%d] = %+v, want summary %q", i, sent[i], want)
}
}
// The fold's own window — a minute from its first mail, 75s — ends at 135s.
clock.Advance(55 * time.Second) // now at 80s + 55s = 135s
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 5 {
t.Fatalf("sent %d notice(s), want 5 (75s and 80s folded into one, at 135s): %+v", len(sent), sent)
}
if sent[4]["summary"] != "2 more new mails in home" || !strings.Contains(sent[4]["body"].(string), "Sender 6") {
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 6 (80s, the latest)", sent[4])
}
}
// TestASteadyTrickleNeverFoldsForever is review round 3 of issue 800's second test: the bug the round found
// (a session that only resets after a quiet minute folds a steady stream forever, even without a burst). One
// mail every 50 seconds, 20 of them: each one's preceding 60 seconds holds only the one before it, so N never
// exceeds 2, and all 20 are shown at once — none ever fold.
func TestASteadyTrickleNeverFoldsForever(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
for i := 1; i <= 20; i++ {
if i > 1 {
clock.Advance(50 * time.Second)
}
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
t.Fatalf("Add mail %d: %v", i, err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush at mail %d: %v", i, err)
}
if len(sent) != i {
t.Fatalf("after mail %d, sent %d notice(s), want %d: a steady trickle must never fold: %+v", i, len(sent), i, sent)
}
}
for i, got := range sent {
want := fmt.Sprintf("Mail %d", i+1)
if got["summary"] != want {
t.Errorf("sent[%d] = %+v, want summary %q", i, got, want)
}
}
}
// TestFiveMailsWithinTenSecondsFoldAfterThree is review round 3 of issue 800's third test: five mails within
// ten seconds — the first three shown at once, the fourth opens a fold, the fifth joins it, and the fold is
// shown 60 seconds after the fourth (the fold's own first mail) as one notice.
func TestFiveMailsWithinTenSecondsFoldAfterThree(t *testing.T) {
clock := &fakeClock{now: time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)}
store := newFakeStore()
var sent []map[string]any
n := newTestNotifier(t, clock, store, &sent)
account := "home"
for i := 1; i <= 5; i++ {
if i > 1 {
clock.Advance(2 * time.Second)
}
if err := n.Add(mail(account, i, fmt.Sprintf("m%d", i), fmt.Sprintf("Mail %d", i), fmt.Sprintf("Sender %d", i))); err != nil {
t.Fatalf("Add mail %d: %v", i, err)
}
if err := n.Flush(); err != nil {
t.Fatalf("Flush at mail %d: %v", i, err)
}
}
if len(sent) != 3 {
t.Fatalf("sent %d notice(s), want 3 shown one by one, the fourth and fifth folded, not yet due: %+v", len(sent), sent)
}
// The fold is due 60 seconds after its own first mail (the fourth, at 6s).
clock.Advance(Window)
if err := n.Flush(); err != nil {
t.Fatalf("Flush: %v", err)
}
if len(sent) != 4 {
t.Fatalf("sent %d notice(s), want 4 (one fold notice carrying the fourth and fifth): %+v", len(sent), sent)
}
if sent[3]["summary"] != "2 more new mails in home" || !strings.Contains(sent[3]["body"].(string), "Sender 5") {
t.Errorf("the fold notice is %+v, want it to count only what it holds (2), naming Sender 5 (the latest)", sent[3])
}
}
@@ -0,0 +1,114 @@
package main
import (
"errors"
"io"
"strings"
"sync/atomic"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// TestRunStartsTheWatcherEvenWhenStdinIsAtEOF proves the watcher runs beside stdio.Serve in the one process the
// node's runtime launches (novox/hq ADR 0378 §9, as mesh-knowledge's serve starts go hear(k) and go
// background(k) before stdio.Serve), not only in a separate process something keeps asking over stdin. run is
// given stdin already at EOF, exactly what systemd gave the old separate `notify` process: it exited at once,
// every five seconds (novox/hq issue 859). The watcher must still be running once run has returned — proved by
// making it actually answer after run returns, not by the absence of a signal that nothing but the test's own
// cleanup could ever raise.
func TestRunStartsTheWatcherEvenWhenStdinIsAtEOF(t *testing.T) {
started := make(chan struct{})
ping := make(chan struct{})
pong := make(chan struct{})
fakeWatcher := func(n *Notifier) {
close(started)
for range ping {
pong <- struct{}{}
}
}
n := &Notifier{Store: newFakeStore()}
n.init()
done := make(chan error, 1)
go func() { done <- run(n, fakeWatcher, strings.NewReader(""), io.Discard, nil) }()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("the watcher was not started before stdio.Serve returned")
}
select {
case err := <-done:
if err != nil {
t.Fatalf("run over stdin at EOF: %v", err)
}
case <-time.After(time.Second):
t.Fatal("run did not return once stdin reached EOF, as systemd gives a process nobody talks to")
}
// run has returned (stdio.Serve hit EOF at once); the watcher must still be live and answering, not merely
// not yet torn down. A watcher that stopped when run returned would never take this ping.
select {
case ping <- struct{}{}:
case <-time.After(time.Second):
t.Fatal("the watcher is no longer running once run returned: it must not stop just because stdio.Serve did")
}
select {
case <-pong:
case <-time.After(time.Second):
t.Fatal("the watcher did not answer once run returned")
}
close(ping)
}
// TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits proves watch no longer exits the process on a
// subscribe the runtime refuses — the old os.Exit(1) took mail-notifier.status and mail_notifier_health down
// with it — but retries with backoff and logs each refusal, as modules/mesh-knowledge's hear does (novox/hq
// issue 859, review round 1). A subscribe that fails twice then succeeds must leave the process serving (run
// still returns normally over stdin at EOF, never os.Exit) and the watcher subscribed (its third, successful
// call).
func TestWatchRetriesASubscribeRefusalWithBackoffAndNeverExits(t *testing.T) {
var calls int32
subscribed := make(chan struct{})
fakeSubscribe := func(pattern string, handle func(stdio.Envelope) error) error {
if atomic.AddInt32(&calls, 1) <= 2 {
return errors.New("the runtime refused this subscribe")
}
close(subscribed)
return nil
}
noSleep := func(time.Duration) {}
n := &Notifier{Store: newFakeStore()}
n.init()
done := make(chan error, 1)
go func() {
done <- run(n, func(n *Notifier) { watchWith(n, fakeSubscribe, noSleep) },
strings.NewReader(""), io.Discard, nil)
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("run over stdin at EOF: %v", err)
}
case <-time.After(time.Second):
t.Fatal("run did not return once stdin reached EOF: a subscribe refusal must not hold up or kill the process")
}
select {
case <-subscribed:
case <-time.After(time.Second):
t.Fatal("the watcher never subscribed after its first two refusals")
}
if got := atomic.LoadInt32(&calls); got != 3 {
t.Fatalf("subscribe was called %d time(s), want 3 (two refusals, then the one that succeeded)", got)
}
}
@@ -0,0 +1,17 @@
package main
import "testing"
// TestSendKeyNamesThisMachine proves Send asks its own machine's node-notifier, not the bare seat: a bare
// "seat:node-notifier.send" is answered by whichever machine's node-notifier takes it first, so notices sent
// from g14 and shanks were lost to "503 no responders" or another machine's desktop (novox/hq issue 865).
// desk-channel names the machine it asks the same way ("seat:node-notifier.send@"+machine,
// modules/desk-channel/cmd/desk-channel/main.go); sendKey must read this machine's own name the same way the
// SDK's own Emit does, from MESH_NODE, which the runtime sets.
func TestSendKeyNamesThisMachine(t *testing.T) {
t.Setenv("MESH_NODE", "g14")
if got, want := sendKey(), "seat:node-notifier.send@g14"; got != want {
t.Fatalf("sendKey() = %q, want %q (it must name this machine, as desk-channel does)", got, want)
}
}
+5
View File
@@ -0,0 +1,5 @@
module mesh-mail-notifier
go 1.22
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44
+2
View File
@@ -0,0 +1,2 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44 h1:r5xIo5qWOXQdCSMxvqJy8kV4U+P/QWXFK6971Ha6wU8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261013182924-5933eb757d44/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
+67
View File
@@ -0,0 +1,67 @@
{
"module": "mesh-mail-notifier",
"version": "1",
"runs-as": "mesh-mail-notifier",
"claims": [
{
"name": "mail-notifier",
"scope": "node",
"serves": ["status"]
}
],
"consumes": [
"mesh-mail.mail.*.message.added"
],
"invokes": [
"seat:node-notifier.send",
"seat:mail-store.accounts"
],
"emits": [
"shown"
],
"state": [
{
"name": "shown",
"ttl-seconds": 2592000
}
],
"tools": [
"mail-notifier.status",
"mail_notifier_health"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-mail-notifier",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-mail-notifier",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-mail-notifier"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-mail-notifier",
"place": "."
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-mail-notifier",
"binary": "mesh-mail-notifier",
"loads": [
"mesh-mail-notifier"
]
}
]
}
}
+81
View File
@@ -0,0 +1,81 @@
# mesh-mail — the operator's mail accounts, watched and said on the bus
One module holds the operator's mail accounts, watches them, and says each change of a message on the bus for the
modules the operator grants it to (novox/hq ADR 0359, research 049 document 02). One instance, on the control-node.
Reading is the first stage: accounts of the kind `imap`, read-only — nothing here flags, moves, deletes or drafts.
Sending is `mail_send` alone (novox/hq ADR 0361 §2): it sends nothing itself, and reaches an account's outgoing
server only once the operator approves it on a channel that proves who answered.
## Its seats (novox/hq ADR 0378 as amended by ADR 0379, issue 800 part A)
It declares three mail seats and claims the one it holds:
- **mail-store** (held once for the mesh) — serves `accounts`, `search`, `read` and `send`; mesh-mail claims it,
through the same handlers as its own `mail_*` tools. The catalogue's own check (`checks/mailstore`) refuses any
other manifest that claims it.
- **mail-webclient** (held once for the mesh) — serves `open` and `status`; unheld until issue 580.
- **mail-notifier** (held once per machine, needing `node-notifier` there) — serves `status`; held by
mesh-mail-notifier (issue 800 part B), a module of its own built from this one's source, which tells the
operator of each new inbox mail through `node-notifier.send`.
## What it does
- **Watches.** For each account it keeps one connection: IDLE on the inbox, ended and begun again every two minutes
(which proves the server still answers), and a look at every other folder every five minutes. Folders are opened
read-only and bodies are peeked, so watching never marks a message seen.
- **Indexes.** One row per message and folder in its own database (`postgres-database`): metadata, flags, the text
for search, and the attachments' references. Each attachment is kept once, by its SHA-256, in its bucket
(`s3-bucket`, from `minio`).
- **Says each change.** A message added, its flags changed, or removed is said as the event
`mail.<account>.message.<added|changed|removed>`: account, folder, uid, message id, date, sender, recipients,
subject, size, flags and attachment references; never the body, never an attachment's bytes. The events go to the
module's own stream `MAIL`, kept seven days, never to the shared EVENTS stream (novox/hq ADR 0363); a module hears an
account's part only when its `consumes` names mesh-mail and the setting `grants` names it for that account. The first look at a
folder indexes what is there and says nothing: the bus hears what changed since the watch began. The keeper writes
each event in the same step as the change it says; the bundle says it through the runtime and removes it once the
bus took it.
## Its settings, both trusted
`accounts` — per account name (a lower-case token, which names the account on the bus and its secret):
```json
{"mesh": {"kind": "imap", "address": "<the address>",
"incoming": {"host": "<the IMAP server>", "port": 993, "security": "tls"},
"outgoing": {"host": "<the SMTP server>", "port": 465, "security": "tls"},
"folders": ["optional: the folders watched besides the inbox; every folder when left out"]}}
```
`outgoing` is the server `mail_send` sends through: host, port, and `security` `tls` (465, implicit) or `starttls`
(587). A second `imap` account on another server is one more entry and one more password; no code changes. `user`
signs in under another name than the address.
`grants` — per account, who may read its mail, and who may send from it: a module's name (on every machine),
`<machine>.<module>`, or `person.<name>`. Nobody by default. It restricts the stream `MAIL` on the bus (a person's
entry grants nothing there) and the tools `mail_search`, `mail_read` and `mail_send` — the same check for all three.
The password of an account is the own secret `mail-credential-<account>`, given at the desk; the mesh never makes one.
## Its tools
| Tool | Answers |
|---|---|
| `mail_accounts` | each account's kind, state and why, when its server last answered, its folders; the address only to a caller granted it |
| `mail_search` | messages of the accounts the caller is granted, by every word of a query, newest first, no body |
| `mail_read` | one message's metadata and text, or one attachment's bytes, to a caller granted its account |
| `mail_health` | healthy; unhealthy naming the account and why (connection down, or no answer for five minutes); waits naming `mail-credential-<account>` while its password is not given |
| `mail_send` | raises an ask on the operator channel (the account, the recipients, the subject, the first lines) for a caller granted the account; answers at once with the ask's id. Sends nothing itself |
| `mail_send_status` | where a send stands, by its ask id: waiting, sent, declined, ended without a choice, or not proven |
The caller is the runtime's word, never an argument. `mail_send` sends through the account's outgoing server only
once the operator's Approve comes back on a channel that proves who answered, and once: a warrant delivered twice
never sends twice (novox/hq ADR 0361 §2).
The mail-store seat's four verbs, `mail-store.accounts`, `mail-store.search`, `mail-store.read` and
`mail-store.send`, answer through the same handlers as `mail_accounts`, `mail_search`, `mail_read` and `mail_send`
above — the same grants, the same refusals, the same ask-then-Approve for a send.
## Its processes
`prepare` brings the index's schema to this version's (numbered migrations, and `schema.sql` for a fresh database).
`keep` watches the accounts and is judged by its unit. `watch` judges every minute and is judged by `mail_health`.
+260
View File
@@ -0,0 +1,260 @@
package main
// The operator's mail accounts and who may read them (novox/hq ADR 0359 §1, §4, §5): two trusted settings, each a
// file the mesh writes from the module's settings and changes only on the operator's approval. `accounts` says which
// server each account is on and how to sign in (never a credential); `grants` says, per account, which callers may
// read its mail. The password of an account is the module's own secret mail-credential-<account>, given at the desk
// (ADR 0283 §6), at the path the secrets directory and that name make.
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
// The kinds of account an adapter exists for. Only `imap` is built (ADR 0359 §5); `imap-oauth` and `graph` are later
// decisions (§7), and an account of a kind not listed here is refused by name.
const KindIMAP = "imap"
// Kinds is every kind this version has an adapter for.
var Kinds = []string{KindIMAP}
// SecretFamily is the prefix of every account's own secret: mail-credential-<account> (ADR 0283 §6).
const SecretFamily = "mail-credential-"
// accountName is what an account may be called: a member name of the secret family (ADR 0283 §6) and one token of a
// bus subject, so never a dot, a slash or an upper-case letter.
var accountName = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,39}$`)
// Server is where an account's mail is reached.
type Server struct {
Host string `json:"host"`
Port int `json:"port"`
// Security is tls (TLS from the first byte) or starttls (plain, then upgraded before signing in). Nothing
// signs in over a connection that is not encrypted.
Security string `json:"security"`
}
// Address is host:port.
func (s Server) Address() string { return fmt.Sprintf("%s:%d", s.Host, s.Port) }
// Account is one of the operator's mail accounts, as the setting `accounts` declares it.
type Account struct {
// Name is the key the account has in the setting; set when the setting is read.
Name string `json:"-"`
Kind string `json:"kind"`
// Address is the account's mail address, said only to a caller granted the account.
Address string `json:"address"`
// User is the name it signs in with; the address when empty.
User string `json:"user,omitempty"`
Incoming Server `json:"incoming"`
Outgoing *Server `json:"outgoing,omitempty"`
// Folders are the folders watched besides the inbox; every folder the server lists when empty.
Folders []string `json:"folders,omitempty"`
}
// SignInAs is the name the account signs in with.
func (a Account) SignInAs() string {
if a.User != "" {
return a.User
}
return a.Address
}
// Secret is the name of the account's own secret.
func (a Account) Secret() string { return SecretFamily + a.Name }
// AccountsFile is the shape of the trusted setting `accounts`, as the mesh writes it.
type AccountsFile struct {
Accounts map[string]Account `json:"accounts"`
}
// ReadAccounts reads the accounts setting, refusing it whole when any account in it cannot be watched: an account
// half-understood would be watched wrongly, and the operator approved the setting as one.
func ReadAccounts(path string) ([]Account, error) {
if path == "" {
return nil, errors.New("the module was started without its accounts setting (MESH_MAIL_ACCOUNTS)")
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("the accounts setting cannot be read: %w", err)
}
return ParseAccounts(raw)
}
// ParseAccounts reads the accounts setting's content.
func ParseAccounts(raw []byte) ([]Account, error) {
var f AccountsFile
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&f); err != nil {
return nil, fmt.Errorf("the accounts setting is not the module's: %v", err)
}
var out []Account
var problems []string
for name, a := range f.Accounts {
a.Name = name
if err := a.Check(); err != nil {
problems = append(problems, err.Error())
continue
}
out = append(out, a)
}
if len(problems) > 0 {
sort.Strings(problems)
return nil, errors.New("the accounts setting is refused: " + strings.Join(problems, "; "))
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out, nil
}
// Check says why an account cannot be watched, or nil.
func (a Account) Check() error {
say := func(format string, args ...any) error {
return fmt.Errorf("the account %q: %s", a.Name, fmt.Sprintf(format, args...))
}
if !accountName.MatchString(a.Name) {
return say("is not an account's name: a lower-case letter or digit, then lower-case letters, digits and hyphens, at most 40")
}
known := false
for _, k := range Kinds {
known = known || a.Kind == k
}
if !known {
return say("is of the kind %q, and this version watches only %s", a.Kind, strings.Join(Kinds, ", "))
}
if !strings.Contains(a.Address, "@") {
return say("has no mail address")
}
if err := a.Incoming.check("incoming"); err != nil {
return say("%v", err)
}
if a.Outgoing != nil {
if err := a.Outgoing.check("outgoing"); err != nil {
return say("%v", err)
}
}
for _, f := range a.Folders {
if strings.TrimSpace(f) == "" || strings.EqualFold(f, "INBOX") {
return say("lists the folder %q, and the inbox is always watched", f)
}
}
return nil
}
func (s Server) check(which string) error {
switch {
case strings.TrimSpace(s.Host) == "" || strings.ContainsAny(s.Host, " /:"):
return fmt.Errorf("its %s server has no host name", which)
case s.Port <= 0 || s.Port > 65535:
return fmt.Errorf("its %s server's port %d is not a port", which, s.Port)
case s.Security != "tls" && s.Security != "starttls":
return fmt.Errorf("its %s server's security is %q, and only tls or starttls signs in", which, s.Security)
}
return nil
}
// SecretPath is where the account's own secret is placed in the secrets directory, as the manifest declares the
// family: <dir>/mail-credential-<account>.secret.
func SecretPath(dir string, a Account) string { return filepath.Join(dir, a.Secret()+".secret") }
// ReadSecret is the account's password, or given false when nobody gave it yet: the mesh never makes a member of the
// family, so a member not given is no file (ADR 0283 §6).
func ReadSecret(dir string, a Account) (string, bool, error) {
raw, err := os.ReadFile(SecretPath(dir, a))
if errors.Is(err, os.ErrNotExist) {
return "", false, nil
}
if err != nil {
return "", false, err
}
pw := strings.TrimRight(string(raw), "\r\n")
if pw == "" {
return "", false, nil
}
return pw, true, nil
}
// SecretGiven says whether the account's own secret was given.
//
// It looks at the file and never reads it: whoever judges whether a password was given has no need of the password.
// A file that is there and not empty was given; whether it signs in is the keeper's to say.
func SecretGiven(dir string, a Account) bool {
info, err := os.Stat(SecretPath(dir, a))
return err == nil && info.Mode().IsRegular() && info.Size() > 0
}
// ---- grants ------------------------------------------------------------------------------------------------------
// Grants is the trusted setting `grants`: per account, the callers that may read its mail (ADR 0359 §4). An entry is a
// module's name (granted wherever it runs), a module on one machine (`<machine>.<module>`) or a person
// (`person.<name>`), as the runtime names callers.
type Grants map[string][]string
// GrantsFile is the shape of the setting as the mesh writes it.
type GrantsFile struct {
Grants Grants `json:"grants"`
}
// ReadGrants reads the grants setting. A setting that cannot be read grants nothing, and says so.
func ReadGrants(path string) (Grants, error) {
if path == "" {
return Grants{}, errors.New("the module was started without its grants setting (MESH_MAIL_GRANTS)")
}
raw, err := os.ReadFile(path)
if err != nil {
return Grants{}, fmt.Errorf("the grants setting cannot be read: %w", err)
}
var f GrantsFile
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&f); err != nil {
return Grants{}, fmt.Errorf("the grants setting is not the module's: %v", err)
}
if f.Grants == nil {
f.Grants = Grants{}
}
return f.Grants, nil
}
// Unproven is the caller the runtime proved nobody to be; it is granted nothing.
const Unproven = "unproven"
// Allows says whether the caller may read the account's mail. Nobody is granted an account by default, an unproven
// caller never, and a wildcard is no entry.
func (g Grants) Allows(caller, account string) bool {
if caller == "" || caller == Unproven {
return false
}
for _, entry := range g[account] {
if entry == "" || strings.ContainsAny(entry, "*>") {
continue
}
if entry == caller {
return true
}
// A module's name alone grants it on every machine: the caller is <machine>.<module>.
if !strings.Contains(entry, ".") {
if _, module, ok := strings.Cut(caller, "."); ok && module == entry && !strings.HasPrefix(caller, "person.") {
return true
}
}
}
return false
}
// Granted is the accounts, of those given, the caller may read.
func (g Grants) Granted(caller string, accounts []Account) []string {
var out []string
for _, a := range accounts {
if g.Allows(caller, a.Name) {
out = append(out, a.Name)
}
}
return out
}
@@ -0,0 +1,124 @@
package main
import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// The first stage's account, as the operator sets the setting `accounts` (ADR 0359 §5): its address and its server,
// never a credential; its password is the own secret mail-credential-mesh.
const stageOne = `{"accounts": {"mesh": {"kind": "imap", "address": "jschoubben@novox.be",
"incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}}}}`
func TestTheFirstStagesAccountIsReadAndItsSecretNamed(t *testing.T) {
got, err := ParseAccounts([]byte(stageOne))
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Name != "mesh" || got[0].Kind != KindIMAP || got[0].SignInAs() != "jschoubben@novox.be" ||
got[0].Incoming.Address() != "mail.novox.be:993" || got[0].Secret() != "mail-credential-mesh" || got[0].Outgoing != nil {
t.Fatalf("%+v", got)
}
if p := SecretPath("/state", got[0]); p != "/state/mail-credential-mesh.secret" {
t.Fatalf("the secret is placed at %s", p)
}
}
// A second account of the kind imap on another server — Gmail with an app password, another member of the family —
// is configuration only: no code changes for it. Each account may also say its outgoing server, which mail_send
// uses to send through it (novox/hq ADR 0361 §1).
func TestASecondIMAPAccountIsConfigurationOnly(t *testing.T) {
got, err := ParseAccounts([]byte(`{"accounts": {
"mesh": {"kind": "imap", "address": "jschoubben@novox.be", "incoming": {"host": "mail.novox.be", "port": 993, "security": "tls"}},
"gmail": {"kind": "imap", "address": "someone@gmail.com", "incoming": {"host": "imap.gmail.com", "port": 993, "security": "tls"},
"outgoing": {"host": "smtp.gmail.com", "port": 465, "security": "tls"}, "folders": ["[Gmail]/Sent Mail"]}}}`))
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || got[0].Name != "gmail" || got[0].Secret() != "mail-credential-gmail" || got[0].Outgoing.Address() != "smtp.gmail.com:465" ||
!reflect.DeepEqual(got[0].Folders, []string{"[Gmail]/Sent Mail"}) {
t.Fatalf("%+v", got)
}
}
func TestAnAccountThatCannotBeWatchedRefusesTheSetting(t *testing.T) {
for name, raw := range map[string]string{
"a later kind": `{"accounts": {"work": {"kind": "graph", "address": "a@b.c", "incoming": {"host": "h", "port": 1, "security": "tls"}}}}`,
"no encryption": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 143, "security": "none"}}}}`,
"a name with a dot": `{"accounts": {"my.box": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
"a password in it": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "password": "p", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
"no address": `{"accounts": {"x": {"kind": "imap", "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
"the inbox listed": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "folders": ["INBOX"], "incoming": {"host": "h", "port": 993, "security": "tls"}}}}`,
"a bad outgoing one": `{"accounts": {"x": {"kind": "imap", "address": "a@b.c", "incoming": {"host": "h", "port": 993, "security": "tls"}, "outgoing": {"host": "", "port": 465, "security": "tls"}}}}`,
} {
if _, err := ParseAccounts([]byte(raw)); err == nil {
t.Errorf("%s: accepted", name)
}
}
if got, err := ParseAccounts([]byte(`{"accounts": {}}`)); err != nil || len(got) != 0 {
t.Fatalf("the manifest's default: %v %v", got, err)
}
}
// Grants (ADR 0359 §4): nobody by default; a module's name grants it wherever it runs; a module on one machine, or a
// person, by its full name; never the unproven, never by a wildcard, never one account for another.
func TestGrantsAreByAccountAndByProvenCaller(t *testing.T) {
g := Grants{"mesh": {"triage", "novox.desk", "person.jochen", "*"}}
for caller, want := range map[string]bool{
"novox.triage": true, "shanks.triage": true, "novox.desk": true, "g14.desk": false, "person.jochen": true,
"person.triage": false, "novox.other": false, Unproven: false, "": false, "novox.*": false,
} {
if got := g.Allows(caller, "mesh"); got != want {
t.Errorf("%s: %v", caller, got)
}
}
if g.Allows("novox.triage", "gmail") || (Grants{}).Allows("novox.triage", "mesh") {
t.Fatal("a grant reached an account it does not name")
}
dir := t.TempDir()
path := filepath.Join(dir, "grants.json")
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": ["triage"]}}`), 0o600)
read, err := ReadGrants(path)
if err != nil || !read.Allows("novox.triage", "mesh") {
t.Fatalf("%v %v", read, err)
}
_ = os.WriteFile(path, []byte(`{"grants": {"mesh": "triage"}}`), 0o600)
if read, err := ReadGrants(path); err == nil || len(read) != 0 {
t.Fatal("a grants setting it cannot read granted something")
}
}
// Whether a password was given is read from the file's presence and size alone (issue 733): an empty file is not
// given, a file the judge cannot read is still given, and the password is never read to say so.
func TestASecretIsGivenByItsFileNotByReadingIt(t *testing.T) {
dir := t.TempDir()
a := testAccount()
if SecretGiven(dir, a) {
t.Fatal("given before anything was written")
}
_ = os.WriteFile(SecretPath(dir, a), nil, 0o600)
if SecretGiven(dir, a) {
t.Fatal("an empty file is a password")
}
_ = os.WriteFile(SecretPath(dir, a), []byte("pw\n"), 0o600)
if pw, given, _ := ReadSecret(dir, a); !given || pw != "pw" || strings.Contains(pw, "\n") {
t.Fatalf("%q %v", pw, given)
}
// Unreadable to the judge, as a secret owned by root is to the watch: still given.
if err := os.Chmod(SecretPath(dir, a), 0); err != nil {
t.Fatal(err)
}
if !SecretGiven(dir, a) {
t.Fatal("a password the judge cannot read is said not given")
}
src, _ := os.ReadFile("config.go")
body := string(src)
body = body[strings.Index(body, "func SecretGiven("):]
body = body[:strings.Index(body, "\n}\n")]
if strings.Contains(body, "ReadSecret") || strings.Contains(body, "ReadFile") || strings.Contains(body, "os.Open") {
t.Fatalf("SecretGiven reads the password:\n%s", body)
}
}
+170
View File
@@ -0,0 +1,170 @@
package main
// How mesh-mail is (novox/hq ADR 0359 §6). Its keeper's liveness is its unit's; its watching is judged here, by the
// watch process, once a minute, and left in a status file the bundle's tool `mail_health` reads:
//
// - waits, naming the own secret mail-credential-<account>, for an account whose password nobody gave yet (ADR 0283),
// so the condition is needs-operator and the gate passes;
// - unhealthy, naming the account and why, when its connection is down or its server has not answered for
// SilentAfter (five minutes) — an IDLE that went silent;
// - healthy otherwise.
//
// A fault beside a wait is unhealthy and says both.
import (
"context"
"encoding/json"
"fmt"
"os"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
// HealthTool is the module's tool the node-engine asks how it is.
const HealthTool = "mail_health"
// WatchEvery is how often the watch judges; StatusStale how old its verdict may be before the tool says it stopped.
const (
WatchEvery = time.Minute
StatusStale = 3 * WatchEvery
// SignInGrace is how long a password just given may wait for its keeper to sign in before that is a fault.
SignInGrace = 3 * time.Minute
)
// Judge is the module's health from what the watch reads: the accounts setting (or why it cannot be read), whether the
// index serves, which passwords were given, and each account's state as its keeper said it.
func Judge(accounts []Account, accountsErr, readyErr error, given func(Account) bool, states map[string]AccountState, now time.Time) health.Answer {
if accountsErr != nil {
return health.Unhealthy(accountsErr.Error())
}
var problems, fine []string
var waits []health.Wait
if readyErr != nil {
problems = append(problems, "the index cannot serve: "+readyErr.Error())
}
for _, a := range accounts {
if !given(a) {
waits = append(waits, health.Wait{
Part: "the mail account " + a.Name,
Secret: a.Secret(),
What: "the password of the mail account " + a.Name,
})
continue
}
if readyErr != nil {
continue
}
st, said := states[a.Name]
switch {
case !said:
problems = append(problems, fmt.Sprintf("the account %s: its keeper has said nothing of it", a.Name))
case st.State == AccountWaits:
if now.Sub(st.Said) > SignInGrace {
problems = append(problems, fmt.Sprintf("the account %s: its password was given, and its keeper has not signed in for %s",
a.Name, now.Sub(st.Said).Round(time.Second)))
} else {
fine = append(fine, a.Name+" is signing in")
}
case st.State == AccountFailing:
problems = append(problems, fmt.Sprintf("the account %s is not watched: %s", a.Name, st.Why))
case st.Heard.IsZero():
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered since its keeper started", a.Name))
case now.Sub(st.Heard) > SilentAfter:
problems = append(problems, fmt.Sprintf("the account %s: its server has not answered for %s (its IDLE on the inbox is silent)",
a.Name, now.Sub(st.Heard).Round(time.Second)))
default:
fine = append(fine, fmt.Sprintf("%s is %s (%s)", a.Name, st.State, st.Why))
}
}
waitSaid := ""
if len(waits) > 0 {
var names []string
for _, w := range waits {
names = append(names, w.Secret)
}
waitSaid = "waits for its password: " + strings.Join(names, ", ")
}
switch {
case len(problems) > 0:
why := strings.Join(problems, "; ")
if waitSaid != "" {
why += "; and it " + waitSaid
}
return health.Unhealthy(why, waits...)
case len(waits) > 0:
why := "nothing is wrong, and it " + waitSaid
if len(fine) > 0 {
why += "; " + strings.Join(fine, "; ")
}
return health.Waiting(why, waits...)
case len(accounts) == 0:
return health.Healthy("no mail account is declared, so nothing is watched")
}
return health.Healthy(fmt.Sprintf("%d account(s) watched: %s", len(accounts), strings.Join(fine, "; ")))
}
// Status is the watch's verdict as it leaves it: the answer, and when it looked.
type Status struct {
health.Answer
At time.Time `json:"at"`
}
func writeStatus(path string, s Status) error {
raw, err := json.Marshal(s)
if err != nil {
return err
}
tmp := path + ".next"
if err := os.WriteFile(tmp, raw, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
// ReadStatus is the watch's last verdict, as the tool answers it: unhealthy when there is none, when it cannot be read,
// or when it is older than StatusStale.
func ReadStatus(path string, now time.Time) health.Answer {
raw, err := os.ReadFile(path)
if err != nil {
return health.Unhealthy("the watch has left no verdict yet: " + err.Error())
}
var s Status
if err := json.Unmarshal(raw, &s); err != nil {
return health.Unhealthy("the watch's verdict cannot be read")
}
if age := now.Sub(s.At); age > StatusStale {
return health.Unhealthy(fmt.Sprintf("the watch has not looked for %s (its last verdict: %s)", age.Round(time.Second), s.Why))
}
// Read through the SDK's reader, so a wait it cannot check is never excused.
a, err := health.Read(mustJSON(s.Answer))
if err != nil {
return health.Unhealthy("the watch's verdict cannot be read: " + err.Error())
}
return a
}
func mustJSON(v any) []byte {
raw, _ := json.Marshal(v)
return raw
}
// watchOnce is one look: what the watch reads, judged.
func watchOnce(ctx context.Context, env Env, open func(context.Context) (Store, func(), error), now time.Time) health.Answer {
accounts, accountsErr := ReadAccounts(env.Accounts)
var readyErr error
states := map[string]AccountState{}
st, done, err := open(ctx)
if err != nil {
readyErr = err
} else {
defer done()
if readyErr = st.Ready(ctx); readyErr == nil {
if states, err = st.States(ctx); err != nil {
readyErr = err
}
}
}
return Judge(accounts, accountsErr, readyErr, func(a Account) bool { return SecretGiven(env.Secrets, a) }, states, now)
}
@@ -0,0 +1,116 @@
package main
import (
"context"
"encoding/json"
"errors"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/health"
)
var judgedAt = time.Date(2026, 10, 14, 18, 0, 0, 0, time.UTC)
func given(names ...string) func(Account) bool {
return func(a Account) bool { return contains(names, a.Name) }
}
// Before the password is given, mail_health answers waits, naming the secret, which the node-engine reads as a wait
// the controller can excuse (ADR 0283 §1, §3): the family mail-credential-* is declared issued outside the mesh.
func TestItWaitsNamingTheSecretUntilThePasswordIsGiven(t *testing.T) {
a := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
if a.State != health.StateWaits || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-mesh" || a.Healthy {
t.Fatalf("%+v", a)
}
read, err := health.Read(mustJSON(a))
if err != nil || read.State != health.StateWaits {
t.Fatalf("the node-engine reads it as %+v (%v)", read, err)
}
if strings.Contains(a.Waits[0].What, "@") || strings.Contains(a.Why, "@") {
t.Fatalf("the wait says an address: %+v", a)
}
}
// Unhealthy, naming the account and why, when its connection is down or its server silent for five minutes.
func TestItIsUnhealthyWhenAnAccountIsDownOrSilent(t *testing.T) {
acc := []Account{testAccount()}
down := map[string]AccountState{"mesh": {Account: "mesh", State: AccountFailing, Why: "its server mail.example cannot be reached", Said: judgedAt}}
if a := Judge(acc, nil, nil, given("mesh"), down, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "the account mesh is not watched: its server") {
t.Fatalf("down: %+v", a)
}
silent := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Heard: judgedAt.Add(-SilentAfter - time.Second), Said: judgedAt}}
if a := Judge(acc, nil, nil, given("mesh"), silent, judgedAt); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "IDLE on the inbox is silent") {
t.Fatalf("silent: %+v", a)
}
fine := map[string]AccountState{"mesh": {Account: "mesh", State: AccountCurrent, Why: "watching", Heard: judgedAt.Add(-time.Minute), Said: judgedAt}}
if a := Judge(acc, nil, nil, given("mesh"), fine, judgedAt); a.State != health.StateHealthy || !a.Healthy {
t.Fatalf("fine: %+v", a)
}
if a := Judge(acc, nil, errors.New("the schema is behind"), given("mesh"), fine, judgedAt); a.State != health.StateUnhealthy {
t.Fatalf("an index that cannot serve: %+v", a)
}
if a := Judge(nil, errors.New("the accounts setting is refused"), nil, given(), nil, judgedAt); a.State != health.StateUnhealthy {
t.Fatalf("a setting it cannot read: %+v", a)
}
// A fault beside a wait is unhealthy and says both (ADR 0283 §1).
two := []Account{testAccount(), {Name: "gmail", Kind: KindIMAP, Address: "x@example.com", Incoming: Server{Host: "h", Port: 993, Security: "tls"}}}
a := Judge(two, nil, nil, given("mesh"), down, judgedAt)
if a.State != health.StateUnhealthy || len(a.Waits) != 1 || a.Waits[0].Secret != "mail-credential-gmail" || !strings.Contains(a.Why, "mail-credential-gmail") {
t.Fatalf("both: %+v", a)
}
// A password just given is signing in for a while; not for ever.
signing := map[string]AccountState{"mesh": {Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-time.Minute)}}
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateHealthy {
t.Fatalf("signing in: %+v", a)
}
signing["mesh"] = AccountState{Account: "mesh", State: AccountWaits, Said: judgedAt.Add(-SignInGrace - time.Minute)}
if a := Judge(acc, nil, nil, given("mesh"), signing, judgedAt); a.State != health.StateUnhealthy {
t.Fatalf("never signed in: %+v", a)
}
if a := Judge(nil, nil, nil, given(), nil, judgedAt); a.State != health.StateHealthy || !strings.Contains(a.Why, "nothing is watched") {
t.Fatalf("no account: %+v", a)
}
}
// The tool reads the watch's verdict, and says when the watch stopped looking.
func TestTheHealthToolReadsTheWatchsVerdictAndSaysWhenItIsStale(t *testing.T) {
path := filepath.Join(t.TempDir(), "health.json")
if a := ReadStatus(path, judgedAt); a.State != health.StateUnhealthy {
t.Fatalf("no verdict: %+v", a)
}
waits := Judge([]Account{testAccount()}, nil, nil, given(), nil, judgedAt)
if err := writeStatus(path, Status{Answer: waits, At: judgedAt}); err != nil {
t.Fatal(err)
}
if a := ReadStatus(path, judgedAt.Add(time.Minute)); a.State != health.StateWaits || a.Waits[0].Secret != "mail-credential-mesh" {
t.Fatalf("%+v", a)
}
if a := ReadStatus(path, judgedAt.Add(StatusStale+time.Second)); a.State != health.StateUnhealthy || !strings.Contains(a.Why, "has not looked") {
t.Fatalf("stale: %+v", a)
}
}
// One look of the watch, end to end over the files the mesh writes: the accounts setting, the secrets directory and
// the index.
func TestTheWatchJudgesFromTheFilesAndTheIndex(t *testing.T) {
dir := t.TempDir()
env := Env{Accounts: filepath.Join(dir, "accounts.json"), Secrets: dir}
writeFile(t, env.Accounts, stageOne)
st := newMemStore()
open := func(context.Context) (Store, func(), error) { return st, func() {}, nil }
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateWaits {
t.Fatalf("%+v", a)
}
giveSecret(t, dir, Account{Name: "mesh"}, "pw")
_ = st.SetState(context.Background(), AccountState{Account: "mesh", State: AccountCurrent, Heard: judgedAt, Said: judgedAt})
if a := watchOnce(context.Background(), env, open, judgedAt); a.State != health.StateHealthy {
t.Fatalf("%+v", a)
}
raw, _ := json.Marshal(Status{Answer: health.Healthy("x"), At: judgedAt})
if !strings.Contains(string(raw), `"state":"healthy"`) || !strings.Contains(string(raw), `"at":`) {
t.Fatalf("the status file's shape: %s", raw)
}
}
+570
View File
@@ -0,0 +1,570 @@
package main
// The adapter for an account of the kind `imap` (novox/hq ADR 0359 §1, §2, §5). For each account it keeps one
// connection: IDLE on the inbox, and every PollEvery a look at every other folder. Each look compares what the server
// holds with the index and says each difference as an event: a message added, its flags changed, or removed. The first
// look at a folder indexes what is there without saying it — what the bus is told is what changed since the mesh
// started watching, never the history of a mailbox.
//
// **It only reads.** Every folder is opened read-only (EXAMINE), and every message is fetched with BODY.PEEK, so
// watching never marks a message seen. The client is used through reader alone, which has no command that flags,
// moves, copies, deletes, appends or sends (§5); a test holds the code to that.
import (
"context"
"crypto/tls"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
"github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
)
// How the keeper paces itself. IdleCycle is how long one IDLE lasts before it is ended and begun again, which proves
// the server still answers; SilentAfter is how long without an answer makes the account unhealthy (§6).
const (
IdleCycle = 2 * time.Minute
PollEvery = 5 * time.Minute
SilentAfter = 5 * time.Minute
RetryFirst = 30 * time.Second
RetryMost = 5 * time.Minute
FetchBatch = 25
// BatchBytes bounds the mail one fetch of bodies holds; SizeBatch is how many sizes one fetch asks.
BatchBytes = 64 << 20
SizeBatch = 500
)
// Batches cuts the uids, in order, into batches of at most most messages whose sizes add up to at most limit bytes. A
// message larger than limit is a batch of its own. A size not known counts as nothing: the server listed the message,
// and the fetch says what it holds.
func Batches(uids []uint32, sizes map[uint32]int64, limit int64, most int) [][]uint32 {
var out [][]uint32
var cur []uint32
var bytes int64
for _, u := range uids {
size := sizes[u]
if len(cur) > 0 && (len(cur) >= most || bytes+size > limit) {
out = append(out, cur)
cur, bytes = nil, 0
}
cur = append(cur, u)
bytes += size
}
if len(cur) > 0 {
out = append(out, cur)
}
return out
}
func uidSet(uids []uint32) imap.UIDSet {
set := make([]imap.UID, len(uids))
for i, u := range uids {
set[i] = imap.UID(u)
}
return imap.UIDSetNum(set...)
}
func orBytes(n, def int64) int64 {
if n > 0 {
return n
}
return def
}
// reader is the part of an IMAP client the keeper uses: nothing in it changes a mailbox.
type reader interface {
Login(username, password string) *imapclient.Command
List(ref, pattern string, options *imap.ListOptions) *imapclient.ListCommand
Select(mailbox string, options *imap.SelectOptions) *imapclient.SelectCommand
Fetch(numSet imap.NumSet, options *imap.FetchOptions) *imapclient.FetchCommand
Idle() (*imapclient.IdleCommand, error)
Caps() imap.CapSet
Closed() <-chan struct{}
Logout() *imapclient.Command
Close() error
}
// Dialer opens a connection to an account's server, handing what the server says unasked to the handler.
type Dialer func(a Account, handler *imapclient.UnilateralDataHandler) (reader, error)
// DialServer is the dialer for the mesh: TLS from the first byte, or STARTTLS before signing in, as the account says.
func DialServer(a Account, handler *imapclient.UnilateralDataHandler) (reader, error) {
opts := &imapclient.Options{UnilateralDataHandler: handler, TLSConfig: &tls.Config{ServerName: a.Incoming.Host, MinVersion: tls.VersionTLS12}}
var c *imapclient.Client
var err error
switch a.Incoming.Security {
case "tls":
c, err = imapclient.DialTLS(a.Incoming.Address(), opts)
case "starttls":
c, err = imapclient.DialStartTLS(a.Incoming.Address(), opts)
default:
err = fmt.Errorf("the account %s's security %q signs in over nothing this version speaks", a.Name, a.Incoming.Security)
}
if err != nil {
return nil, err
}
return c, nil
}
// Keeper watches the accounts and keeps the index.
type Keeper struct {
Store Store
Objects Objects
Dial Dialer
SecretsDir string
Logf func(format string, a ...any)
Now func() time.Time
// Paces, for the tests; the constants above when zero.
IdleCycle, PollEvery, RetryFirst time.Duration
// BatchBytes, for the tests; the constant when zero.
BatchBytes int64
}
func (k *Keeper) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
func (k *Keeper) logf(format string, a ...any) {
if k.Logf != nil {
k.Logf(format, a...)
}
}
func orDefault(d, def time.Duration) time.Duration {
if d > 0 {
return d
}
return def
}
// Run watches every account until ctx ends.
func (k *Keeper) Run(ctx context.Context, accounts []Account) {
var wg sync.WaitGroup
for _, a := range accounts {
wg.Add(1)
go func(a Account) {
defer wg.Done()
k.Watch(ctx, a)
}(a)
}
wg.Wait()
}
func (k *Keeper) say(ctx context.Context, a Account, state, why string, heard, synced time.Time) {
if err := k.Store.SetState(ctx, AccountState{Account: a.Name, State: state, Why: why, Heard: heard, Synced: synced, Said: k.now()}); err != nil {
k.logf("[mesh-mail] the account %s's state could not be kept: %v", a.Name, err)
}
}
// Watch keeps one account until ctx ends: waits while its secret is not given, signs in, watches, and on any fault
// says it, waits a little longer each time, and begins again.
func (k *Keeper) Watch(ctx context.Context, a Account) {
retry := orDefault(k.RetryFirst, RetryFirst)
var heard, synced time.Time
for ctx.Err() == nil {
pw, given, err := ReadSecret(k.SecretsDir, a)
switch {
case err != nil:
k.say(ctx, a, AccountFailing, "its password was given and cannot be read: "+err.Error(), heard, synced)
case !given:
k.say(ctx, a, AccountWaits, "its password ("+a.Secret()+") has not been given", heard, synced)
default:
s := &session{k: k, a: a, wake: make(chan struct{}, 1), heard: heard, synced: synced}
err = s.run(ctx, pw)
heard, synced = s.heard, s.synced
if ctx.Err() != nil {
return
}
k.say(ctx, a, AccountFailing, err.Error(), heard, synced)
k.logf("[mesh-mail] the account %s: %v; trying again in %s", a.Name, err, retry)
}
select {
case <-ctx.Done():
return
case <-time.After(retry):
}
if given && err != nil {
retry = min(retry*2, RetryMost)
} else {
retry = orDefault(k.RetryFirst, RetryFirst)
}
}
}
// session is one connection to one account's server.
type session struct {
k *Keeper
a Account
c reader
wake chan struct{}
heard time.Time
synced time.Time
// state and why are what the session last said of the account, and told when it said it.
state, why string
told time.Time
}
// answered is the server having answered: kept, and told at least once a minute, so a long first look at a large
// mailbox still shows the account alive to the health watch.
func (s *session) answered() {
s.heard = s.k.now()
if s.state != "" && s.heard.Sub(s.told) >= time.Minute {
s.tell(s.state, s.why)
}
}
// tell says the account's state.
func (s *session) tell(state, why string) {
s.state, s.why, s.told = state, why, s.k.now()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
s.k.say(ctx, s.a, state, why, s.heard, s.synced)
}
func (s *session) poke() {
select {
case s.wake <- struct{}{}:
default:
}
}
func (s *session) run(ctx context.Context, password string) error {
handler := &imapclient.UnilateralDataHandler{
Expunge: func(uint32) { s.poke() },
Mailbox: func(*imapclient.UnilateralDataMailbox) { s.poke() },
Fetch: func(*imapclient.FetchMessageData) { s.poke() },
}
c, err := s.k.Dial(s.a, handler)
if err != nil {
return fmt.Errorf("its server %s cannot be reached: %v", s.a.Incoming.Host, err)
}
s.c = c
defer c.Close()
if err := c.Login(s.a.SignInAs(), password).Wait(); err != nil {
var said *imap.Error
if errors.As(err, &said) {
return fmt.Errorf("its server refused the sign-in (%s): its password may be wrong, and is given again at the desk", strings.TrimSpace(said.Text))
}
return fmt.Errorf("signing in failed: %v", err)
}
s.answered()
s.tell(AccountSyncing, "signed in; looking at its folders")
folders, err := s.folders()
if err != nil {
return err
}
idles := c.Caps().Has(imap.CapIdle) || c.Caps().Has(imap.CapIMAP4rev2)
var polled time.Time
for ctx.Err() == nil {
if polled.IsZero() || s.k.now().Sub(polled) >= orDefault(s.k.PollEvery, PollEvery) {
for _, f := range folders {
if err := s.k.syncFolder(ctx, c, s.a, f, s.answered); err != nil {
return err
}
}
polled = s.k.now()
}
// The inbox last, so it is the folder selected while the session idles.
if err := s.k.syncFolder(ctx, c, s.a, "INBOX", s.answered); err != nil {
return err
}
s.synced = s.k.now()
s.tell(AccountCurrent, fmt.Sprintf("watching the inbox and %d other folder(s)", len(folders)))
if err := s.wait(ctx, idles); err != nil {
return err
}
}
_ = c.Logout().Wait()
return ctx.Err()
}
// wait is one IDLE on the inbox, ended by what the server says unasked, by the cycle's end or by ctx. Ending it waits
// for the server's answer, which is what proves the connection alive.
func (s *session) wait(ctx context.Context, idles bool) error {
cycle := time.NewTimer(orDefault(s.k.IdleCycle, IdleCycle))
defer cycle.Stop()
if !idles {
select {
case <-ctx.Done():
case <-cycle.C:
case <-s.c.Closed():
return errors.New("its server closed the connection")
}
return nil
}
idle, err := s.c.Idle()
if err != nil {
return fmt.Errorf("its server refused to IDLE on the inbox: %v", err)
}
select {
case <-ctx.Done():
case <-s.wake:
case <-cycle.C:
case <-s.c.Closed():
}
if err := idle.Close(); err != nil {
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
}
if err := idle.Wait(); err != nil {
return fmt.Errorf("its IDLE on the inbox ended badly: %v", err)
}
s.answered()
return nil
}
// folders is what is watched besides the inbox: the account's list, or every folder the server can open.
func (s *session) folders() ([]string, error) {
if len(s.a.Folders) > 0 {
return append([]string(nil), s.a.Folders...), nil
}
listed, err := s.c.List("", "*", nil).Collect()
if err != nil {
return nil, fmt.Errorf("its folders cannot be listed: %v", err)
}
s.answered()
var out []string
for _, l := range listed {
if strings.EqualFold(l.Mailbox, "INBOX") || hasAttr(l.Attrs, imap.MailboxAttrNoSelect) || hasAttr(l.Attrs, imap.MailboxAttrNonExistent) {
continue
}
out = append(out, l.Mailbox)
}
sort.Strings(out)
return out, nil
}
func hasAttr(attrs []imap.MailboxAttr, want imap.MailboxAttr) bool {
for _, a := range attrs {
if strings.EqualFold(string(a), string(want)) {
return true
}
}
return false
}
// syncFolder brings the index of one folder to what the server holds, and says each difference, unless this is the
// first look at the folder.
func (k *Keeper) syncFolder(ctx context.Context, c reader, a Account, folder string, answered func()) error {
sel, err := c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait()
if err != nil {
return fmt.Errorf("the folder %s cannot be opened: %v", folder, err)
}
answered()
known, seen, err := k.Store.Folder(ctx, a.Name, folder)
if err != nil {
return err
}
if seen && known.UIDValidity != sel.UIDValidity {
k.logf("[mesh-mail] the account %s's folder %s was renumbered by its server; it is indexed afresh", a.Name, folder)
if err := k.Store.DropFolder(ctx, a.Name, folder); err != nil {
return err
}
seen = false
}
server := map[uint32][]string{}
if sel.NumMessages > 0 {
all, err := c.Fetch(imap.UIDSet{{Start: 1, Stop: 0}}, &imap.FetchOptions{UID: true, Flags: true}).Collect()
if err != nil {
return fmt.Errorf("the folder %s's messages cannot be listed: %v", folder, err)
}
answered()
for _, m := range all {
server[uint32(m.UID)] = flagStrings(m.Flags)
}
}
index, err := k.Store.Flags(ctx, a.Name, folder)
if err != nil {
return err
}
say := func(e *Event) *Event {
if seen {
return e
}
return nil
}
// Removed: in the index, no longer on the server.
for uid := range index {
if _, still := server[uid]; still {
continue
}
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
if err != nil {
return err
}
if m.UID == 0 {
m = Message{Account: a.Name, Folder: folder, UID: uid, UIDValidity: sel.UIDValidity}
}
if err := k.Store.Remove(ctx, a.Name, folder, uid, say(removedEvent(m))); err != nil {
return err
}
}
// Changed: on both, with other flags.
var added []uint32
for uid, flags := range server {
before, indexed := index[uid]
if !indexed {
added = append(added, uid)
continue
}
if sameStrings(before, flags) {
continue
}
m, _, err := k.Store.Get(ctx, a.Name, folder, uid)
if err != nil {
return err
}
m.Flags = flags
if err := k.Store.SetFlags(ctx, a.Name, folder, uid, flags, say(changedEvent(m, before, flags))); err != nil {
return err
}
}
// Added: on the server, not yet in the index. Their sizes first, then their bodies in batches bounded by bytes as
// well as by count, so one batch never holds more than BatchBytes of mail in memory; a message larger than that
// is fetched alone, and said. Fetched without marking them seen.
sort.Slice(added, func(i, j int) bool { return added[i] < added[j] })
sizes := map[uint32]int64{}
for start := 0; start < len(added); start += SizeBatch {
part := added[start:min(start+SizeBatch, len(added))]
got, err := c.Fetch(uidSet(part), &imap.FetchOptions{UID: true, RFC822Size: true}).Collect()
if err != nil {
return fmt.Errorf("the folder %s's new messages cannot be measured: %v", folder, err)
}
answered()
for _, m := range got {
sizes[uint32(m.UID)] = m.RFC822Size
}
}
cap := orBytes(k.BatchBytes, BatchBytes)
for _, batch := range Batches(added, sizes, cap, FetchBatch) {
if len(batch) == 1 && sizes[batch[0]] > cap {
k.logf("[mesh-mail] the account %s's message %d in %s is %d bytes, more than a batch holds; it is fetched alone",
a.Name, batch[0], folder, sizes[batch[0]])
}
msgs, err := c.Fetch(uidSet(batch), &imap.FetchOptions{
UID: true, Flags: true, Envelope: true, RFC822Size: true, InternalDate: true,
BodySection: []*imap.FetchItemBodySection{{Peek: true}},
}).Collect()
if err != nil {
return fmt.Errorf("the folder %s's new messages cannot be fetched: %v", folder, err)
}
answered()
for _, fm := range msgs {
m, err := k.index(ctx, a, folder, sel.UIDValidity, fm)
if err != nil {
return err
}
if err := k.Store.Add(ctx, m, say(addedEvent(m))); err != nil {
return err
}
}
}
return k.Store.SetFolder(ctx, Folder{Account: a.Name, Name: folder, UIDValidity: sel.UIDValidity, Messages: len(server), Synced: k.now()})
}
// index is a fetched message as the index keeps it: its metadata, its text, and its attachments kept in the store.
func (k *Keeper) index(ctx context.Context, a Account, folder string, validity uint32, fm *imapclient.FetchMessageBuffer) (Message, error) {
m := Message{Account: a.Name, Folder: folder, UIDValidity: validity, UID: uint32(fm.UID), Size: fm.RFC822Size,
Flags: flagStrings(fm.Flags), Date: fm.InternalDate.UTC(), From: []Person{}, To: []Person{}, Attachments: []AttachmentRef{}}
if env := fm.Envelope; env != nil {
m.MessageID, m.Subject = env.MessageID, env.Subject
if len(env.InReplyTo) > 0 {
m.InReplyTo = env.InReplyTo[0]
}
if !env.Date.IsZero() {
m.Date = env.Date.UTC()
}
m.From, m.To, m.Cc = people(env.From), people(env.To), people(env.Cc)
}
var raw []byte
if len(fm.BodySection) > 0 {
raw = fm.BodySection[0].Bytes
}
parsed, err := ParseMessage(raw)
if err != nil {
k.logf("[mesh-mail] the account %s's message %d in %s is read in part: %v", a.Name, m.UID, folder, err)
}
m.Body = parsed.Text
for _, p := range parsed.Attachments {
ref, err := Keep(ctx, k.Objects, p.Data, p.Type, p.Name)
if err != nil {
return m, err
}
m.Attachments = append(m.Attachments, ref)
}
return m, nil
}
func people(as []imap.Address) []Person {
out := []Person{}
for _, a := range as {
if addr := a.Addr(); addr != "" {
out = append(out, Person{Name: a.Name, Address: addr})
}
}
return out
}
func flagStrings(fs []imap.Flag) []string {
out := make([]string, 0, len(fs))
for _, f := range fs {
if strings.EqualFold(string(f), `\Recent`) {
continue
}
out = append(out, string(f))
}
sort.Strings(out)
return out
}
func sameStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// ---- the events ---------------------------------------------------------------------------------------------------
// EventKey and Changes, the event's shape, now live in mailevent.go of their own (issue 800 part A).
// metadata is what an event says of a message: never its body, never an attachment's bytes (ADR 0359 §2).
func metadata(m Message) map[string]any {
atts := make([]map[string]any, 0, len(m.Attachments))
for _, r := range m.Attachments {
atts = append(atts, map[string]any{"bucket": r.Bucket, "key": r.Key, "size": r.Size, "type": r.Type, "name": r.Name})
}
return map[string]any{
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity,
"message-id": m.MessageID, "in-reply-to": m.InReplyTo, "date": m.Date.UTC().Format(time.RFC3339),
"from": m.From, "to": m.To, "cc": m.Cc, "subject": m.Subject, "size": m.Size, "flags": m.Flags,
"attachments": atts,
}
}
func addedEvent(m Message) *Event {
return &Event{Key: EventKey(m.Account, "added"), Body: metadata(m)}
}
func changedEvent(m Message, before, after []string) *Event {
body := metadata(m)
body["changes"] = map[string]any{"flags": map[string]any{"before": before, "after": after}}
return &Event{Key: EventKey(m.Account, "changed"), Body: body}
}
func removedEvent(m Message) *Event {
return &Event{Key: EventKey(m.Account, "removed"), Body: map[string]any{
"account": m.Account, "folder": m.Folder, "uid": m.UID, "uidvalidity": m.UIDValidity, "message-id": m.MessageID,
}}
}
@@ -0,0 +1,300 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"testing"
"time"
"github.com/emersion/go-imap/v2"
)
// keeping starts a keeper on one account of the test server, its paces short, and stops it when the test ends.
func keeping(t *testing.T, s *mailServer, st Store, objects Objects, secrets string) {
t.Helper()
k := &Keeper{Store: st, Objects: objects, Dial: s.dialer(), SecretsDir: secrets, Logf: t.Logf,
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { k.Watch(ctx, testAccount()); close(done) }()
t.Cleanup(func() { cancel(); <-done })
}
func stateOf(st Store, account string) AccountState {
states, _ := st.States(context.Background())
return states[account]
}
// float64OrUint reads a uid from an event body whether it came from memory (uint32) or from postgres (float64).
type float64OrUint struct{ v any }
func (f float64OrUint) value() uint32 {
switch n := f.v.(type) {
case uint32:
return n
case float64:
return uint32(n)
}
return 0
}
func uidOf(e Event) uint32 { return float64OrUint{e.Body["uid"]}.value() }
func eventsNamed(st Store, key string) []Event {
evs, _ := st.Pending(context.Background(), 1000)
var out []Event
for _, e := range evs {
if e.Key == key {
out = append(out, e)
}
}
return out
}
func describe(st Store) string {
evs, _ := st.Pending(context.Background(), 1000)
var keys []string
for _, e := range evs {
keys = append(keys, e.Key)
}
s := stateOf(st, "mesh")
return "state " + s.State + " (" + s.Why + "); events " + strings.Join(keys, ", ")
}
// The whole watch, over a real IMAP conversation (ADR 0359 §2, §3, §5): the first look indexes what is there and says
// nothing; then a new mail in the inbox is said within moments of its delivery, through IDLE, with its metadata and
// its attachment by reference, never its body; its flags changed are said; its removal is said; a mail in another
// folder is said at the next poll; and watching never marks a message seen.
func TestTheKeeperSaysEachChangeOnTheBusAndOnlyReads(t *testing.T) {
eachStore(t, func(t *testing.T, st Store) {
s := newMailServer(t)
op := s.operator(t)
deliver(t, op, "INBOX", plainMail("before the watch", "an old letter"))
deliver(t, op, "Archive", plainMail("archived long ago", "an older letter"))
secrets := t.TempDir()
giveSecret(t, secrets, testAccount(), testPassword)
objects := newMemObjects()
keeping(t, s, st, objects, secrets)
eventually(t, 10*time.Second, "the first look", func() (bool, string) {
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
})
ctx := context.Background()
if hits, _ := st.Search(ctx, Query{Text: "letter", Accounts: []string{"mesh"}}); len(hits) != 2 {
t.Fatalf("the first look indexed %d messages, not 2", len(hits))
}
if evs, _ := st.Pending(ctx, 100); len(evs) != 0 {
t.Fatalf("the first look said %d event(s): the history of a mailbox is not news", len(evs))
}
// A new mail with an attachment, delivered while the keeper idles on the inbox.
deliver(t, op, "INBOX", attachedMail("the invoice"))
var added Event
eventually(t, 5*time.Second, "the new mail said", func() (bool, string) {
evs := eventsNamed(st, "mail.mesh.message.added")
if len(evs) == 1 {
added = evs[0]
}
return len(evs) == 1, describe(st)
})
body := added.Body
if body["subject"] != "the invoice" || body["folder"] != "INBOX" || body["account"] != "mesh" || body["message-id"] != "att@example.org" {
t.Fatalf("the event says %v", body)
}
raw, _ := json.Marshal(body)
for _, never := range []string{"Please find", "dGhlIGludm9pY2Ug", attachmentData} {
if strings.Contains(string(raw), never) {
t.Fatalf("the event carries the body or an attachment's bytes (%q): %s", never, raw)
}
}
sum := sha256.Sum256([]byte(attachmentData))
key := hex.EncodeToString(sum[:])
if !strings.Contains(string(raw), `"key":"`+key+`"`) || !strings.Contains(string(raw), `"name":"invoice.pdf"`) {
t.Fatalf("the event does not name the attachment by its SHA-256 %s: %s", key, raw)
}
if kept, _ := objects.Get(ctx, key); string(kept) != attachmentData {
t.Fatalf("the attachment store keeps %q under %s", kept, key)
}
uid := uidOf(added)
m, ok, _ := st.Get(ctx, "mesh", "INBOX", uid)
if !ok || !strings.Contains(m.Body, "Please find the invoice attached.") {
t.Fatalf("the index keeps the text for mail_read: %+v", m)
}
// Flagged by the operator elsewhere: said as changed, with the flags before and after.
if _, err := op.Select("INBOX", nil).Wait(); err != nil {
t.Fatal(err)
}
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagFlagged}, Silent: true}, nil).Close(); err != nil {
t.Fatal(err)
}
eventually(t, 5*time.Second, "the flag said", func() (bool, string) {
evs := eventsNamed(st, "mail.mesh.message.changed")
if len(evs) != 1 {
return false, describe(st)
}
raw, _ := json.Marshal(evs[0].Body["changes"])
return strings.Contains(string(raw), `"after":["\\Flagged"]`), string(raw)
})
// Never marked seen by the watching: the message is still unseen on the server.
fetched, err := op.Fetch(imap.UIDSetNum(imap.UID(uid)), &imap.FetchOptions{Flags: true, UID: true}).Collect()
if err != nil || len(fetched) != 1 {
t.Fatalf("%v %v", fetched, err)
}
for _, f := range fetched[0].Flags {
if f == imap.FlagSeen {
t.Fatal("watching marked the message seen")
}
}
// Deleted by the operator elsewhere: said as removed.
if err := op.Store(imap.UIDSetNum(imap.UID(uid)), &imap.StoreFlags{Op: imap.StoreFlagsAdd, Flags: []imap.Flag{imap.FlagDeleted}, Silent: true}, nil).Close(); err != nil {
t.Fatal(err)
}
if err := op.Expunge().Close(); err != nil {
t.Fatal(err)
}
eventually(t, 5*time.Second, "the removal said", func() (bool, string) {
evs := eventsNamed(st, "mail.mesh.message.removed")
return len(evs) == 1 && uidOf(evs[0]) == uid && evs[0].Body["message-id"] == "att@example.org", describe(st)
})
if _, ok, _ := st.Get(ctx, "mesh", "INBOX", uid); ok {
t.Fatal("the removed message is still indexed")
}
// A mail in another folder is said at the next poll.
deliver(t, op, "Archive", plainMail("filed away", "a letter filed"))
eventually(t, 5*time.Second, "the archived mail said", func() (bool, string) {
for _, e := range eventsNamed(st, "mail.mesh.message.added") {
if e.Body["folder"] == "Archive" && e.Body["subject"] == "filed away" {
return true, ""
}
}
return false, describe(st)
})
if s := stateOf(st, "mesh"); s.State != AccountCurrent || time.Since(s.Heard) > time.Minute {
t.Fatalf("the account is %+v", s)
}
})
}
// No password given: the account waits, and nothing is dialled. A wrong one: the account fails, saying the server
// refused the sign-in. Given later: it is picked up without a restart.
func TestTheKeeperWaitsForThePasswordAndSaysARefusal(t *testing.T) {
s := newMailServer(t)
st := newMemStore()
secrets := t.TempDir()
keeping(t, s, st, newMemObjects(), secrets)
eventually(t, 5*time.Second, "the wait", func() (bool, string) {
got := stateOf(st, "mesh")
return got.State == AccountWaits && strings.Contains(got.Why, "mail-credential-mesh"), describe(st)
})
giveSecret(t, secrets, testAccount(), "not-the-password")
eventually(t, 5*time.Second, "the refusal", func() (bool, string) {
got := stateOf(st, "mesh")
return got.State == AccountFailing && strings.Contains(got.Why, "refused the sign-in"), describe(st)
})
giveSecret(t, secrets, testAccount(), testPassword)
eventually(t, 10*time.Second, "the sign-in", func() (bool, string) {
return stateOf(st, "mesh").State == AccountCurrent, describe(st)
})
}
// It only reads a mailbox (ADR 0359 §5): no code of the module, but its tests, issues a command that flags, moves,
// copies, deletes, appends, creates, renames or otherwise writes to one over IMAP, and every folder is opened
// read-only and every body peeked. mail_send is the one exception the operator's own Approve carves out (ADR 0361
// §2): it never touches a mailbox either — it speaks SMTP, in smtp.go alone, to the account's outgoing server, and
// only once the operator approved it (send.go).
func TestNothingButTheTestsWritesToAMailbox(t *testing.T) {
files, err := filepath.Glob("*.go")
if err != nil {
t.Fatal(err)
}
writes := regexp.MustCompile(`([A-Za-z_][A-Za-z0-9_]*)\.(Store|Move|Copy|Expunge|UIDExpunge|Append|Create|Delete|Rename|Subscribe|Unsubscribe|SetMetadata|SetQuota)\(`)
for _, f := range files {
if strings.HasSuffix(f, "_test.go") || f == "send.go" || f == "smtp.go" {
// Neither touches a mailbox: send.go keeps the `sends` state on the bus, never IMAP; smtp.go speaks SMTP
// to the account's outgoing server, never to a mailbox, and only on the operator's Approve (ADR 0361 §2).
continue
}
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
for _, m := range writes.FindAllSubmatch(raw, -1) {
// os is a file of the module's own, never a mailbox; stdio is the bus SDK (main.go's hear subscribes to
// the router's own decided subject, exactly as mesh-knowledge does, novox/hq issue 790 review round 1)
// — its Subscribe has nothing to do with an IMAP mailbox's.
if string(m[1]) != "os" && string(m[1]) != "stdio" {
t.Errorf("%s calls %s, which writes to a mailbox", f, m[0])
}
}
if strings.Contains(string(raw), "net/smtp") {
t.Errorf("%s speaks SMTP: only smtp.go does, and only on the operator's Approve (ADR 0361 §2)", f)
}
}
src, _ := os.ReadFile("imap.go")
if !strings.Contains(string(src), "&imap.SelectOptions{ReadOnly: true}") || !strings.Contains(string(src), "{{Peek: true}}") {
t.Fatal("folders are not opened read-only, or bodies not peeked")
}
}
// A fetch of bodies is bounded by bytes (issue 733): batches cut by size and by count, and a message larger than a
// batch holds alone.
func TestBatchesAreBoundedByBytesAndALargerMessageGoesAlone(t *testing.T) {
sizes := map[uint32]int64{1: 40, 2: 40, 3: 500, 4: 10, 5: 10, 6: 10}
got := Batches([]uint32{1, 2, 3, 4, 5, 6}, sizes, 100, 2)
want := [][]uint32{{1, 2}, {3}, {4, 5}, {6}}
if fmt.Sprint(got) != fmt.Sprint(want) {
t.Fatalf("%v, not %v", got, want)
}
if got := Batches(nil, nil, 100, 25); len(got) != 0 {
t.Fatalf("%v", got)
}
}
// Over a real conversation: with a cap of a few hundred bytes, a large mail is fetched alone and said so, and every
// mail is still indexed and said.
func TestTheKeeperFetchesUnderAByteCap(t *testing.T) {
s := newMailServer(t)
op := s.operator(t)
st := newMemStore()
secrets := t.TempDir()
giveSecret(t, secrets, testAccount(), testPassword)
var mu sync.Mutex
var said []string
k := &Keeper{Store: st, Objects: newMemObjects(), Dial: s.dialer(), SecretsDir: secrets,
Logf: func(f string, a ...any) { mu.Lock(); said = append(said, fmt.Sprintf(f, a...)); mu.Unlock() },
IdleCycle: 300 * time.Millisecond, PollEvery: 400 * time.Millisecond, RetryFirst: 100 * time.Millisecond, BatchBytes: 600}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { k.Watch(ctx, testAccount()); close(done) }()
t.Cleanup(func() { cancel(); <-done })
eventually(t, 10*time.Second, "the first look", func() (bool, string) { return stateOf(st, "mesh").State == AccountCurrent, describe(st) })
deliver(t, op, "INBOX", plainMail("small one", "short"))
deliver(t, op, "INBOX", plainMail("large one", strings.Repeat("a long letter ", 200)))
deliver(t, op, "INBOX", plainMail("small two", "short"))
eventually(t, 5*time.Second, "three mails said", func() (bool, string) {
return len(eventsNamed(st, "mail.mesh.message.added")) == 3, describe(st)
})
mu.Lock()
defer mu.Unlock()
alone := 0
for _, line := range said {
if strings.Contains(line, "fetched alone") {
alone++
}
}
if alone != 1 {
t.Fatalf("said %d message(s) fetched alone: %v", alone, said)
}
}
+252
View File
@@ -0,0 +1,252 @@
package main
// What the tests stand on: an IMAP server in this process (go-imap's own in-memory server, speaking the protocol over
// a real socket), the index in memory always and on a real postgres when one can be had (MESH_TEST_POSTGRES, or a
// private server started here from the machine's initdb on a socket of its own, listening on no port), and the
// attachment store in memory.
import (
"context"
"fmt"
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
"github.com/emersion/go-imap/v2/imapserver"
"github.com/emersion/go-imap/v2/imapserver/imapmemserver"
)
func TestMain(m *testing.M) {
code := m.Run()
stopPostgres()
os.Exit(code)
}
// ---- the IMAP server --------------------------------------------------------------------------------------------
const (
testUser = "operator@example.org"
testPassword = "the-password"
)
type mailServer struct {
addr string
srv *imapserver.Server
}
// newMailServer is a server holding one user with an inbox and an Archive folder.
func newMailServer(t *testing.T) *mailServer {
t.Helper()
mem := imapmemserver.New()
user := imapmemserver.NewUser(testUser, testPassword)
for _, f := range []string{"INBOX", "Archive"} {
if err := user.Create(f, nil); err != nil {
t.Fatal(err)
}
}
mem.AddUser(user)
srv := imapserver.New(&imapserver.Options{
NewSession: func(*imapserver.Conn) (imapserver.Session, *imapserver.GreetingData, error) {
return mem.NewSession(), nil, nil
},
InsecureAuth: true,
Caps: imap.CapSet{imap.CapIMAP4rev1: {}, imap.CapIMAP4rev2: {}},
})
ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
go func() { _ = srv.Serve(ln) }()
t.Cleanup(func() { _ = srv.Close() })
return &mailServer{addr: ln.Addr().String(), srv: srv}
}
// dialer reaches the test server without TLS, which only a test does: the mesh's dialer signs in over TLS alone.
func (s *mailServer) dialer() Dialer {
return func(_ Account, h *imapclient.UnilateralDataHandler) (reader, error) {
c, err := imapclient.DialInsecure(s.addr, &imapclient.Options{UnilateralDataHandler: h})
if err != nil {
return nil, err
}
return c, nil
}
}
// operator is a second client, as the operator's mail program or another sender is: it appends, flags and deletes.
func (s *mailServer) operator(t *testing.T) *imapclient.Client {
t.Helper()
c, err := imapclient.DialInsecure(s.addr, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = c.Close() })
if err := c.Login(testUser, testPassword).Wait(); err != nil {
t.Fatal(err)
}
return c
}
func deliver(t *testing.T, c *imapclient.Client, folder, raw string) {
t.Helper()
raw = strings.ReplaceAll(raw, "\n", "\r\n")
cmd := c.Append(folder, int64(len(raw)), nil)
if _, err := cmd.Write([]byte(raw)); err != nil {
t.Fatal(err)
}
if err := cmd.Close(); err != nil {
t.Fatal(err)
}
if _, err := cmd.Wait(); err != nil {
t.Fatal(err)
}
}
func plainMail(subject, body string) string {
return "From: Alice <alice@example.org>\nTo: Operator <operator@example.org>\nSubject: " + subject +
"\nMessage-Id: <" + strings.ReplaceAll(subject, " ", "-") + "@example.org>\nDate: Wed, 14 Oct 2026 10:00:00 +0000\n" +
"MIME-Version: 1.0\nContent-Type: text/plain; charset=utf-8\n\n" + body + "\n"
}
// attachmentData is an attachment's content, and attachedMail a message carrying it.
const attachmentData = "the invoice, in full"
func attachedMail(subject string) string {
return "From: Bob <bob@example.org>\nTo: operator@example.org\nSubject: " + subject + "\nMessage-Id: <att@example.org>\n" +
"Date: Wed, 14 Oct 2026 11:00:00 +0000\nMIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=XYZ\n\n" +
"--XYZ\nContent-Type: text/plain; charset=utf-8\n\nPlease find the invoice attached.\n" +
"--XYZ\nContent-Type: application/pdf; name=invoice.pdf\nContent-Disposition: attachment; filename=invoice.pdf\n" +
"Content-Transfer-Encoding: base64\n\ndGhlIGludm9pY2UsIGluIGZ1bGw=\n--XYZ--\n"
}
// ---- the account ------------------------------------------------------------------------------------------------
func testAccount() Account {
return Account{Name: "mesh", Kind: KindIMAP, Address: testUser, Incoming: Server{Host: "localhost", Port: 993, Security: "tls"}}
}
func giveSecret(t *testing.T, dir string, a Account, pw string) {
t.Helper()
if err := os.WriteFile(SecretPath(dir, a), []byte(pw+"\n"), 0o600); err != nil {
t.Fatal(err)
}
}
// eventually waits for a condition, failing with what it last saw.
func eventually(t *testing.T, within time.Duration, what string, ok func() (bool, string)) {
t.Helper()
deadline := time.Now().Add(within)
last := ""
for time.Now().Before(deadline) {
var done bool
if done, last = ok(); done {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("%s did not happen within %s; last seen: %s", what, within, last)
}
// ---- postgres ---------------------------------------------------------------------------------------------------
var (
pgOnce sync.Once
pgBase string
pgDir string
pgWhy string
pgCount int
pgMu sync.Mutex
)
func startPostgres() {
if u := os.Getenv("MESH_TEST_POSTGRES"); u != "" {
pgBase = u
return
}
initdb, err := exec.LookPath("initdb")
if err != nil {
pgWhy = "no postgres: MESH_TEST_POSTGRES is unset and the machine has no initdb"
return
}
pgctl := filepath.Join(filepath.Dir(initdb), "pg_ctl")
dir, err := os.MkdirTemp("", "mmpg")
if err != nil {
pgWhy = err.Error()
return
}
pgDir = dir
if out, err := exec.Command(initdb, "-D", dir+"/data", "-U", "postgres", "-A", "trust", "-E", "UTF8", "--no-locale").CombinedOutput(); err != nil {
pgWhy = fmt.Sprintf("initdb: %v: %s", err, out)
return
}
if out, err := exec.Command(pgctl, "-D", dir+"/data", "-l", dir+"/log", "-w", "-o",
"-k "+dir+" -c listen_addresses='' -F", "start").CombinedOutput(); err != nil {
pgWhy = fmt.Sprintf("pg_ctl start: %v: %s", err, out)
return
}
pgBase = "postgres://postgres@/postgres?host=" + dir
}
func stopPostgres() {
if pgDir == "" {
return
}
if initdb, err := exec.LookPath("initdb"); err == nil {
_ = exec.Command(filepath.Join(filepath.Dir(initdb), "pg_ctl"), "-D", pgDir+"/data", "-m", "immediate", "stop").Run()
}
_ = os.RemoveAll(pgDir)
}
// freshDatabase is a database of its own for one test, or a skip saying why there is none.
func freshDatabase(t *testing.T) string {
t.Helper()
pgOnce.Do(startPostgres)
if pgBase == "" {
t.Skipf("NOT TESTED ON POSTGRES: %s", pgWhy)
}
pgMu.Lock()
pgCount++
name := fmt.Sprintf("mail_%d_%d", os.Getpid(), pgCount)
pgMu.Unlock()
ctx := context.Background()
admin, err := OpenPgStore(ctx, pgBase)
if err != nil {
t.Fatal(err)
}
defer admin.Close()
if _, err := admin.pool.Exec(ctx, "create database "+name); err != nil {
t.Fatal(err)
}
u, err := url.Parse(pgBase)
if err != nil {
t.Fatal(err)
}
u.Path = "/" + name
return u.String()
}
func pgStore(t *testing.T) *PgStore {
t.Helper()
s, err := OpenPgStore(context.Background(), freshDatabase(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(s.Close)
if _, err := s.Prepare(context.Background()); err != nil {
t.Fatal(err)
}
return s
}
// eachStore runs a test over the index in memory and on postgres.
func eachStore(t *testing.T, run func(t *testing.T, s Store)) {
t.Run("memory", func(t *testing.T) { run(t, newMemStore()) })
t.Run("postgres", func(t *testing.T) { run(t, pgStore(t)) })
}
@@ -0,0 +1,13 @@
package main
// The shape of a mail-store message event (novox/hq ADR 0378 §2, issue 800 part A), moved here on its own so
// mesh-mail-notifier (issue 800 part B) carries a byte-identical copy, the way modules/networkmanager's
// uplink-tools carries ifwatchd's own file (novox/hq ADR 0241, copies_test.go). Nothing here reaches a mailbox or
// a server; it only says where an account's change is told on the bus and what the three changes are called.
// EventKey is where an account's change is said: mail.<account>.message.<added|changed|removed> (research 049
// document 02 §5), the account by its name in the setting, never its address.
func EventKey(account, change string) string { return "mail." + account + ".message." + change }
// Changes are the three things said of a message.
var Changes = []string{"added", "changed", "removed"}
+291
View File
@@ -0,0 +1,291 @@
// mesh-mail: the operator's mail accounts, watched, and each mail said on the bus (novox/hq ADR 0359). One instance,
// on the control-node. Reading is read-only: no flag, move, delete or draft. mail_send is the one way out (novox/hq
// ADR 0361 §2): it sends nothing itself, and reaches an account's outgoing server only on the operator's Approve.
//
// mesh-mail the bundle the node's runtime launches: the tools mail_accounts, mail_search, mail_read,
// mail_health, mail_send and mail_send_status, and the events the keeper wrote, said on the
// bus as the module
// mesh-mail prepare the preparation step: the index's schema brought to this version's
// mesh-mail keep acts: watches every account (IDLE on the inbox, a look at the other folders every five
// minutes), keeps the index and the attachment store, and writes each change's event beside it
// mesh-mail watch judges: reads every minute how each account is, and leaves its verdict for mail_health
//
// The two long-running processes are the act and the judge, as mesh-knowledge's are: keep is judged by its unit, watch
// by the tool mail_health (ADR 0359 §6, ADR 0240 §3 and §7). Only the bundle reaches the bus, through the runtime that
// launched it, so the keeper hands its events over in the index, written in the same step as the change they say,
// and the bundle says them and removes each once the bus took it. stdout is the bundle's MCP channel; what this module
// says, it says on stderr.
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"os/signal"
"sync"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// Module is this module's name.
const Module = "mesh-mail"
// Env is where the mesh puts what the module reads, as the manifest's environment names it.
type Env struct {
Accounts string // the trusted setting accounts
Grants string // the trusted setting grants
Secrets string // the directory the own secrets mail-credential-<account> are placed in
Store string // the attachment store's facts
Status string // the watch's verdict
}
func envFromOS() Env {
return Env{
Accounts: os.Getenv("MESH_MAIL_ACCOUNTS"),
Grants: os.Getenv("MESH_MAIL_GRANTS"),
Secrets: os.Getenv("MESH_MAIL_SECRETS"),
Store: os.Getenv("MESH_MAIL_STORE"),
Status: os.Getenv("MESH_MAIL_STATUS"),
}
}
func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) }
func main() {
if len(os.Args) > 1 {
switch os.Args[1] {
case "prepare":
os.Exit(prepare())
case "keep":
os.Exit(keep(envFromOS()))
case "watch":
os.Exit(watch(envFromOS()))
}
}
serve(envFromOS())
}
// ---- prepare ------------------------------------------------------------------------------------------------------
func prepare() int {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
s, err := PgStoreFromEnv(ctx)
if err != nil {
logf("[mesh-mail] preparing the index: %v", err)
return 1
}
defer s.Close()
said, err := s.Prepare(ctx)
for _, line := range said {
logf("[mesh-mail] %s", line)
}
if err != nil {
logf("[mesh-mail] preparing the index failed: %v", err)
return 1
}
return 0
}
// ---- keep ---------------------------------------------------------------------------------------------------------
// keep watches every account until it is stopped. A setting it cannot read, an index that is behind or a store it
// cannot reach ends it with a fault, which its unit says; the mesh starts it again.
func keep(env Env) int {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
accounts, err := ReadAccounts(env.Accounts)
if err != nil {
logf("[mesh-mail] %v", err)
return 1
}
st, err := PgStoreFromEnv(ctx)
if err != nil {
logf("[mesh-mail] the index cannot be opened: %v", err)
return 1
}
defer st.Close()
if err := st.Ready(ctx); err != nil {
logf("[mesh-mail] %v", err)
return 1
}
facts, err := ReadStoreFacts(env.Store)
if err != nil {
logf("[mesh-mail] %v", err)
return 1
}
k := &Keeper{Store: st, Objects: NewS3(facts), Dial: DialServer, SecretsDir: env.Secrets, Logf: logf}
logf("[mesh-mail] keeping %d account(s)", len(accounts))
k.Run(ctx, accounts)
return 0
}
// ---- watch --------------------------------------------------------------------------------------------------------
func watch(env Env) int {
if env.Status == "" {
logf("[mesh-mail] watch was started without MESH_MAIL_STATUS")
return 2
}
open := func(ctx context.Context) (Store, func(), error) {
s, err := PgStoreFromEnv(ctx)
if err != nil {
return nil, nil, err
}
return s, s.Close, nil
}
for {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
now := time.Now().UTC()
answer := watchOnce(ctx, env, open, now)
cancel()
if err := writeStatus(env.Status, Status{Answer: answer, At: now}); err != nil {
logf("[mesh-mail] the verdict could not be written: %v", err)
}
time.Sleep(WatchEvery)
}
}
// ---- the bundle ---------------------------------------------------------------------------------------------------
func serve(env Env) {
var mu sync.Mutex
var st *PgStore
b := &Bundle{
Env: env,
Index: func(ctx context.Context) (Store, error) {
mu.Lock()
defer mu.Unlock()
if st == nil {
s, err := PgStoreFromEnv(ctx)
if err != nil {
return nil, err
}
st = s
}
if err := st.Ready(ctx); err != nil {
return nil, err
}
return st, nil
},
Objects: func() (Objects, error) {
facts, err := ReadStoreFacts(env.Store)
if err != nil {
return nil, err
}
return NewS3(facts), nil
},
}
b.Logf = logf
p := &Publisher{Index: b.Index, Emit: func(key string, body any) error { return stdio.Emit(key, body) }, Logf: logf}
go p.Run(context.Background())
go hear(b)
go sweepSends(b)
if err := stdio.Serve("", b.Tools()); err != nil {
logf("%v", err)
os.Exit(1)
}
}
// hear subscribes to the router's word on this module's own asks, asked again until the runtime takes it, exactly
// as mesh-knowledge does. Each warrant heard goes to Heard, the same Decide that mail_send_status uses: an
// approved send goes out as soon as the operator answers it, never only when something later calls
// mail_send_status (novox/hq ADR 0361 §2, review round 1 of issue 790).
func hear(b *Bundle) {
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
time.Sleep(wait)
err := stdio.Subscribe(asks.Seat+".decided."+Module, func(e stdio.Envelope) error {
var w asks.Warrant
if json.Unmarshal(e.Body, &w) == nil {
b.Heard(w)
}
return nil
})
if err == nil {
logf("[mesh-mail] hears the operator's answers")
return
}
logf("[mesh-mail] does not hear the operator's answers yet (%v); asking again in %s", err, wait)
}
}
// sweepSends reads every send still undecided once a minute and asks the router's own record for its answer, so
// an Approve answered while this module could not hear it directly is still acted on (novox/hq issue 790, review
// round 1).
func sweepSends(b *Bundle) {
tick := time.NewTicker(SweepEvery)
defer tick.Stop()
for {
b.Sweep(time.Now())
<-tick.C
}
}
// Publisher says the events the keeper wrote, oldest first, and removes each once the bus took it. One the bus did not
// take stays, and is said again on the next round: delivery is at least once, and a consumer reads the uid and the
// folder to know a repeat.
type Publisher struct {
Index func(ctx context.Context) (Store, error)
Emit func(key string, body any) error
Logf func(format string, a ...any)
Every time.Duration
}
// PublishBatch is how many events one round says at most.
const PublishBatch = 100
// Once says what is pending, and answers how many it said.
func (p *Publisher) Once(ctx context.Context) (int, error) {
st, err := p.Index(ctx)
if err != nil {
return 0, err
}
pending, err := st.Pending(ctx, PublishBatch)
if err != nil {
return 0, err
}
var said []int64
var failed error
for _, e := range pending {
if err := p.Emit(e.Key, e.Body); err != nil {
failed = fmt.Errorf("the bus did not take %s: %v", e.Key, err)
break
}
said = append(said, e.ID)
}
if len(said) > 0 {
if err := st.Said(ctx, said); err != nil {
return 0, err
}
}
return len(said), failed
}
// Run says pending events until ctx ends: every few seconds, at once again while a round was full.
func (p *Publisher) Run(ctx context.Context) {
every := p.Every
if every <= 0 {
every = 3 * time.Second
}
// The runtime takes the bundle's first calls only once Serve runs.
time.Sleep(every)
for ctx.Err() == nil {
c, cancel := context.WithTimeout(ctx, time.Minute)
n, err := p.Once(c)
cancel()
if err != nil && p.Logf != nil {
p.Logf("[mesh-mail] events wait to be said: %v", err)
}
if n == PublishBatch {
continue
}
select {
case <-ctx.Done():
case <-time.After(every):
}
}
}
@@ -0,0 +1,182 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
)
// The manifest says what the code does (novox/hq ADR 0359): the database and the bucket it requires (§3), the own
// secret family issued outside the mesh (§5, ADR 0283 §6), the two trusted settings (§4), the tools (§5), the events,
// its two long-running processes with their health (§6), and — ADR 0378 as amended by ADR 0379, issue 800 part A —
// the three mail seats it declares and the one, mail-store, it claims. It is assigned to no machine by itself.
type seat struct {
Name string `json:"name"`
Scope string `json:"scope"`
Serves []string `json:"serves"`
Needs []string `json:"needs"`
}
type manifest struct {
Module string `json:"module"`
Slug string `json:"slug"`
Seats []seat `json:"seats"`
Requires []string `json:"requires"`
Contributes map[string]map[string]any `json:"contributes"`
OwnSecrets map[string]map[string]string `json:"own-secrets"`
Tools []string `json:"tools"`
Emits []string `json:"emits"`
Streams []struct {
Name string `json:"name"`
Events []string `json:"events"`
MaxAgeDays int `json:"max-age-days"`
Grant struct {
Config string `json:"config"`
Events string `json:"events"`
} `json:"grant"`
} `json:"streams"`
Consumes []string `json:"consumes"`
Invokes []string `json:"invokes"`
Claims []seat `json:"claims"`
Resources []map[string]any `json:"resources"`
Build struct {
Artifacts []struct {
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
Env map[string]string `json:"env"`
} `json:"artifacts"`
} `json:"build"`
}
func readManifest(t *testing.T) (manifest, map[string]map[string]any) {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
var m manifest
if err := json.Unmarshal(raw, &m); err != nil {
t.Fatal(err)
}
byID := map[string]map[string]any{}
for _, r := range m.Resources {
byID[r["id"].(string)] = r
}
return m, byID
}
func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) {
m, byID := readManifest(t)
if m.Module != Module || !reflect.DeepEqual(m.Requires, []string{"postgres-database", "s3-bucket"}) || m.Contributes["postgres-database"]["name"] != "mail" {
t.Fatalf("module %s, requires %v, contributes %v", m.Module, m.Requires, m.Contributes)
}
// mesh_<node>_<slug> fits the S3 access key's 20 characters on every node.
if len("mesh_shanks_"+m.Slug) > 20 {
t.Fatalf("the slug %q overflows an S3 access key", m.Slug)
}
fam := m.OwnSecrets[SecretFamily+"*"]
if fam["issued-by"] != "outside" || fam["path"] != "${dir:state}/"+SecretFamily+"*.secret" || len(m.OwnSecrets) != 1 {
t.Fatalf("own secrets %v", m.OwnSecrets)
}
var names []string
for _, tool := range (&Bundle{}).Tools() {
names = append(names, tool.Name)
}
wantTools := []string{
"mail_accounts", "mail_search", "mail_read", "mail_health", "mail_send", "mail_send_status",
"mail-store.accounts", "mail-store.search", "mail-store.read", "mail-store.send",
}
if !reflect.DeepEqual(m.Tools, names) || !reflect.DeepEqual(names, wantTools) {
t.Fatalf("tools %v, the code serves %v", m.Tools, names)
}
// Its events go to its own stream MAIL, kept a week, each account's part heard only by the consumers the trusted
// config value grants names for it (ADR 0359 §2, §4; ADR 0363): none on the shared EVENTS stream.
var events []string
for _, c := range Changes {
events = append(events, EventKey("*", c))
}
if len(m.Emits) != 0 || len(m.Streams) != 1 || m.Streams[0].Name != "mail" || !reflect.DeepEqual(m.Streams[0].Events, events) ||
m.Streams[0].MaxAgeDays != 7 || m.Streams[0].Grant.Config != "grants" || m.Streams[0].Grant.Events != "mail.{key}.**" {
t.Fatalf("emits %v, streams %+v; the code says %v", m.Emits, m.Streams, events)
}
// It calls nothing and consumes nothing: mesh-mail-notifier (issue 800 part B) is the one that invokes and
// consumes, not the store.
if len(m.Consumes)+len(m.Invokes) != 0 {
t.Fatalf("consumes %v, invokes %v", m.Consumes, m.Invokes)
}
// It declares exactly the three mail seats (ADR 0378 §1 as amended by ADR 0379 §4) — mail-store and
// mail-webclient each held once for the mesh, mail-notifier held once per machine and needing node-notifier
// there — and claims only mail-store, serving the same four verbs as its own tools (§2, §3).
wantSeats := []seat{
{Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}},
{Name: "mail-webclient", Scope: "mesh", Serves: []string{"open", "status"}},
{Name: "mail-notifier", Scope: "node", Serves: []string{"status"}, Needs: []string{"node-notifier"}},
}
if !reflect.DeepEqual(m.Seats, wantSeats) {
t.Fatalf("seats %+v, want %+v", m.Seats, wantSeats)
}
wantClaims := []seat{
{Name: "mail-store", Scope: "mesh", Serves: []string{"accounts", "search", "read", "send"}},
}
if !reflect.DeepEqual(m.Claims, wantClaims) {
t.Fatalf("claims %+v, want %+v", m.Claims, wantClaims)
}
// The two settings are trusted, and their defaults are an empty account list and no grant.
for id, want := range map[string]string{"accounts": `{"accounts":{}}`, "grants": `{"grants":{}}`} {
r := byID[id]
if r == nil || r["trusted"] != true || r["merge"] != "json" || !reflect.DeepEqual(r["takes-settings"], []any{id}) {
t.Fatalf("%s: %v", id, r)
}
if strings.Join(strings.Fields(r["content"].(string)), "") != want {
t.Fatalf("%s's default is %q", id, r["content"])
}
if _, err := ParseAccounts([]byte(byID["accounts"]["content"].(string))); err != nil {
t.Fatal(err)
}
}
a := m.Build.Artifacts
if len(a) != 1 || a[0].From != "cmd/mesh-mail" || a[0].Binary != Module || !reflect.DeepEqual(a[0].Loads, []string{Module}) {
t.Fatalf("build %+v", m.Build)
}
for k, v := range map[string]string{"MESH_MAIL_ACCOUNTS": "accounts", "MESH_MAIL_GRANTS": "grants", "MESH_MAIL_STORE": "store", "DATABASE_URL_FILE": "database-url"} {
if a[0].Env[k] != byID[v]["path"] {
t.Errorf("the bundle reads %s at %s, and the manifest writes %s at %v", k, a[0].Env[k], v, byID[v]["path"])
}
}
if a[0].Env["MESH_MAIL_SECRETS"] != "${dir:state}" {
t.Errorf("the secrets directory is %s", a[0].Env["MESH_MAIL_SECRETS"])
}
}
// Its processes: the preparation once, keep judged by its unit, watch by mail_health (ADR 0359 §6), the watch writing
// the verdict the bundle's tool reads, as the account the bundle runs as.
func TestItsProcessesAreThePreparationTheActAndTheJudge(t *testing.T) {
m, byID := readManifest(t)
prep, keep, watch := byID["prepare"], byID["keep"], byID["watch"]
if prep == nil || prep["run-once"] != true || prep["run"].([]any)[1] != "prepare" {
t.Fatalf("prepare %v", prep)
}
if keep == nil || keep["run"].([]any)[1] != "keep" || keep["health"].(map[string]any)["kind"] != "unit" {
t.Fatalf("keep %v", keep)
}
h, _ := watch["health"].(map[string]any)
if watch == nil || watch["run"].([]any)[1] != "watch" || h["kind"] != "tool" || h["tool"] != HealthTool || h["needs"] != "postgres-database" {
t.Fatalf("watch %v", watch)
}
if watch["user"] != "${machine:account}" || watch["env"].(map[string]any)["MESH_MAIL_STATUS"] != m.Build.Artifacts[0].Env["MESH_MAIL_STATUS"] {
t.Fatalf("watch writes another status than the tool reads, or as another account: %v", watch)
}
// keep runs as the machine's root, which the own secrets' files belong to; it starts again when its settings or
// its store change.
if _, as := keep["user"]; as {
t.Fatalf("keep runs as %v", keep["user"])
}
if !reflect.DeepEqual(keep["restart-on"], []any{"database-url", "store", "accounts"}) {
t.Fatalf("keep restarts on %v", keep["restart-on"])
}
}
@@ -0,0 +1,46 @@
-- mesh-mail's index (novox/hq ADR 0359 §3): one row per message and folder, what is known of each folder, each
-- account's state as its keeper said it, and the events written with the change they say, until they are said.
create table folder (
account text not null,
name text not null,
uidvalidity bigint not null,
messages integer not null default 0,
synced timestamptz not null default now(),
primary key (account, name));
create table message (
account text not null,
folder text not null,
uid bigint not null,
uidvalidity bigint not null,
message_id text not null default '',
in_reply_to text not null default '',
date timestamptz not null,
sender jsonb not null default '[]',
recipients jsonb not null default '[]',
copied jsonb not null default '[]',
people text not null default '',
subject text not null default '',
size bigint not null default 0,
flags text[] not null default '{}',
attachments jsonb not null default '[]',
body text not null default '',
search tsvector generated always as (
to_tsvector('simple'::regconfig, subject || ' ' || people || ' ' || body)) stored,
primary key (account, folder, uid));
create index message_search on message using gin (search);
create index message_date on message (account, date desc);
create table account_state (
account text primary key,
state text not null,
why text not null default '',
heard timestamptz,
synced timestamptz,
said timestamptz not null default now());
create table outbox (
id bigserial primary key,
key text not null,
body jsonb not null,
made timestamptz not null default now());
+125
View File
@@ -0,0 +1,125 @@
package main
// A message's text and its attachments, read from its MIME source. The text is kept in the index for search and served
// through mail_read; each attachment goes to the attachment store once, by its SHA-256 (novox/hq ADR 0359 §3).
import (
"bytes"
"errors"
"html"
"io"
"mime"
"regexp"
"strings"
_ "github.com/emersion/go-message/charset" // the charsets mail is written in, beyond UTF-8 and ASCII
"github.com/emersion/go-message/mail"
)
// Part is one attachment as read from a message.
type Part struct {
Name string
Type string
Data []byte
}
// MostBodyText bounds the text kept per message: a longer body is cut, and says so.
const MostBodyText = 256 << 10
// Parsed is a message's text and its attachments.
type Parsed struct {
Text string
Attachments []Part
}
// ParseMessage reads a message's text (its plain part; else its HTML part as text) and its attachments. A message the
// reader cannot follow keeps what was read before the fault, and the error says what.
func ParseMessage(raw []byte) (Parsed, error) {
var p Parsed
r, err := mail.CreateReader(bytes.NewReader(raw))
if r == nil {
return p, err
}
var plain, htmlText string
for {
part, err := r.NextPart()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return finish(p, plain, htmlText), err
}
switch h := part.Header.(type) {
case *mail.InlineHeader:
ct, params, _ := h.ContentType()
_, disp, _ := h.ContentDisposition()
body, readErr := io.ReadAll(io.LimitReader(part.Body, 64<<20))
if readErr != nil {
return finish(p, plain, htmlText), readErr
}
name := params["name"]
if name == "" {
name = disp["filename"]
}
switch {
case ct == "text/plain" && plain == "" && name == "":
plain = string(body)
case ct == "text/html" && htmlText == "" && name == "":
htmlText = HTMLText(string(body))
case !strings.HasPrefix(ct, "text/") || name != "":
// An inline image or a named inline file is an attachment too.
p.Attachments = append(p.Attachments, Part{Name: name, Type: orOctets(ct), Data: body})
}
case *mail.AttachmentHeader:
ct, _, _ := h.ContentType()
name, _ := h.Filename()
body, readErr := io.ReadAll(io.LimitReader(part.Body, 64<<20))
if readErr != nil {
return finish(p, plain, htmlText), readErr
}
p.Attachments = append(p.Attachments, Part{Name: name, Type: orOctets(ct), Data: body})
}
}
return finish(p, plain, htmlText), nil
}
func finish(p Parsed, plain, htmlText string) Parsed {
p.Text = plain
if strings.TrimSpace(p.Text) == "" {
p.Text = htmlText
}
p.Text = strings.ToValidUTF8(p.Text, "�")
if len(p.Text) > MostBodyText {
p.Text = strings.ToValidUTF8(p.Text[:MostBodyText], "") + "\n[the text goes on; it was cut at 256 KiB]"
}
return p
}
func orOctets(ct string) string {
if ct == "" {
return "application/octet-stream"
}
if t, _, err := mime.ParseMediaType(ct); err == nil {
return t
}
return ct
}
var (
scriptsAndStyles = regexp.MustCompile(`(?is)<(script|style|head)\b.*?</(script|style|head)>`)
breaks = regexp.MustCompile(`(?i)<(br|/p|/div|/tr|/li|/h[1-6])\b[^>]*>`)
tags = regexp.MustCompile(`(?s)<[^>]*>`)
blankLines = regexp.MustCompile(`\n[ \t]*\n([ \t]*\n)+`)
)
// HTMLText is an HTML body as plain text: no script, style or markup, the entities read, paragraphs kept as lines.
// Text only, so nothing of the sender's markup ever reaches whoever reads it.
func HTMLText(s string) string {
s = scriptsAndStyles.ReplaceAllString(s, "")
s = breaks.ReplaceAllString(s, "\n")
s = tags.ReplaceAllString(s, "")
s = html.UnescapeString(s)
s = strings.ReplaceAll(s, "\r", "")
s = blankLines.ReplaceAllString(s, "\n\n")
return strings.TrimSpace(s)
}
+275
View File
@@ -0,0 +1,275 @@
package main
// The attachment store (novox/hq ADR 0359 §3): each attachment kept once, by its SHA-256, in the bucket the `minio`
// module provides this module through the `s3-bucket` provision. The mesh writes the bucket's facts and its key into
// one file (MESH_MAIL_STORE); this speaks S3 to it with the standard library alone: a HEAD, a PUT and a GET, each
// signed with AWS Signature Version 4, path-style, as an S3-compatible object store answers them.
import (
"bytes"
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"sort"
"strings"
"sync"
"time"
)
// Objects is the attachment store.
type Objects interface {
// Bucket is the bucket's name, as an event names it.
Bucket() string
// Has says whether the key is kept.
Has(ctx context.Context, key string) (bool, error)
// Put keeps the bytes under the key.
Put(ctx context.Context, key, mediaType string, data []byte) error
// Get is the bytes kept under the key.
Get(ctx context.Context, key string) ([]byte, error)
}
// Keep stores an attachment once by its SHA-256 and answers its reference: the same file mailed ten times is one
// object.
func Keep(ctx context.Context, o Objects, data []byte, mediaType, name string) (AttachmentRef, error) {
sum := sha256.Sum256(data)
key := hex.EncodeToString(sum[:])
has, err := o.Has(ctx, key)
if err != nil {
return AttachmentRef{}, err
}
if !has {
if err := o.Put(ctx, key, mediaType, data); err != nil {
return AttachmentRef{}, err
}
}
return AttachmentRef{Bucket: o.Bucket(), Key: key, Size: int64(len(data)), Type: mediaType, Name: name}, nil
}
// StoreFacts is the file the mesh writes from the s3-bucket binding and its secret.
type StoreFacts struct {
Scheme string `json:"scheme"`
At string `json:"at"`
Port string `json:"port"`
Bucket string `json:"bucket"`
Region string `json:"region"`
Key string `json:"key"`
Secret string `json:"secret"`
}
// ReadStoreFacts reads the store's facts.
func ReadStoreFacts(path string) (StoreFacts, error) {
var f StoreFacts
if path == "" {
return f, errors.New("the module was started without its store's facts (MESH_MAIL_STORE)")
}
raw, err := os.ReadFile(path)
if err != nil {
return f, fmt.Errorf("the store's facts cannot be read: %w", err)
}
if err := json.Unmarshal(raw, &f); err != nil {
return f, fmt.Errorf("the store's facts are not the module's: %v", err)
}
if f.At == "" || f.Bucket == "" || f.Key == "" || f.Secret == "" {
return f, errors.New("the store's facts name no address, bucket or key")
}
if f.Scheme == "" {
f.Scheme = "http"
}
if f.Region == "" {
f.Region = "us-east-1"
}
return f, nil
}
// S3 is the attachment store over S3.
type S3 struct {
Endpoint string // scheme://host[:port]
Name string
Region string
Access string
Secret string
Client *http.Client
Now func() time.Time
}
// NewS3 is the store the facts name.
func NewS3(f StoreFacts) *S3 {
host := f.At
if f.Port != "" {
host += ":" + f.Port
}
return &S3{Endpoint: f.Scheme + "://" + host, Name: f.Bucket, Region: f.Region, Access: f.Key, Secret: f.Secret,
Client: &http.Client{Timeout: 2 * time.Minute}, Now: time.Now}
}
func (s *S3) Bucket() string { return s.Name }
func (s *S3) do(ctx context.Context, method, key, mediaType string, body []byte) (*http.Response, error) {
u, err := url.Parse(s.Endpoint + "/" + s.Name + "/" + key)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, method, u.String(), bytes.NewReader(body))
if err != nil {
return nil, err
}
if mediaType != "" {
req.Header.Set("Content-Type", mediaType)
}
req.ContentLength = int64(len(body))
Sign(req, body, s.Access, s.Secret, s.Region, s.Now().UTC())
return s.Client.Do(req)
}
func (s *S3) Has(ctx context.Context, key string) (bool, error) {
resp, err := s.do(ctx, http.MethodHead, key, "", nil)
if err != nil {
return false, fmt.Errorf("the attachment store cannot be reached: %w", err)
}
resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
return true, nil
case http.StatusNotFound:
return false, nil
}
return false, fmt.Errorf("the attachment store answered %s to asking for %s", resp.Status, key)
}
func (s *S3) Put(ctx context.Context, key, mediaType string, data []byte) error {
resp, err := s.do(ctx, http.MethodPut, key, mediaType, data)
if err != nil {
return fmt.Errorf("the attachment store cannot be reached: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
said, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
return fmt.Errorf("the attachment store refused %s: %s %s", key, resp.Status, strings.TrimSpace(string(said)))
}
return nil
}
func (s *S3) Get(ctx context.Context, key string) ([]byte, error) {
resp, err := s.do(ctx, http.MethodGet, key, "", nil)
if err != nil {
return nil, fmt.Errorf("the attachment store cannot be reached: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("the attachment store answered %s for %s", resp.Status, key)
}
return io.ReadAll(resp.Body)
}
// Sign signs a request with AWS Signature Version 4 for S3: every header it carries but the agent and the length is
// signed, with the payload's hash.
func Sign(req *http.Request, body []byte, access, secret, region string, now time.Time) {
payload := sha256.Sum256(body)
payloadHex := hex.EncodeToString(payload[:])
amzDate := now.Format("20060102T150405Z")
day := now.Format("20060102")
req.Header.Set("x-amz-date", amzDate)
req.Header.Set("x-amz-content-sha256", payloadHex)
headers := map[string]string{"host": req.URL.Host}
for k, v := range req.Header {
lk := strings.ToLower(k)
if lk == "user-agent" || lk == "content-length" || lk == "authorization" {
continue
}
headers[lk] = strings.TrimSpace(strings.Join(v, ","))
}
names := make([]string, 0, len(headers))
for k := range headers {
names = append(names, k)
}
sort.Strings(names)
var canonHeaders strings.Builder
for _, k := range names {
canonHeaders.WriteString(k + ":" + headers[k] + "\n")
}
signed := strings.Join(names, ";")
path := req.URL.EscapedPath()
if path == "" {
path = "/"
}
canonical := strings.Join([]string{req.Method, path, canonicalQuery(req.URL.Query()), canonHeaders.String(), signed, payloadHex}, "\n")
scope := day + "/" + region + "/s3/aws4_request"
sum := sha256.Sum256([]byte(canonical))
toSign := "AWS4-HMAC-SHA256\n" + amzDate + "\n" + scope + "\n" + hex.EncodeToString(sum[:])
key := hmacOf([]byte("AWS4"+secret), day)
key = hmacOf(key, region)
key = hmacOf(key, "s3")
key = hmacOf(key, "aws4_request")
sig := hex.EncodeToString(hmacOf(key, toSign))
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+access+"/"+scope+", SignedHeaders="+signed+", Signature="+sig)
}
func canonicalQuery(q url.Values) string {
keys := make([]string, 0, len(q))
for k := range q {
keys = append(keys, k)
}
sort.Strings(keys)
var parts []string
for _, k := range keys {
vs := append([]string(nil), q[k]...)
sort.Strings(vs)
for _, v := range vs {
parts = append(parts, url.QueryEscape(k)+"="+strings.ReplaceAll(url.QueryEscape(v), "+", "%20"))
}
}
return strings.Join(parts, "&")
}
func hmacOf(key []byte, data string) []byte {
h := hmac.New(sha256.New, key)
h.Write([]byte(data))
return h.Sum(nil)
}
// ---- in memory ----------------------------------------------------------------------------------------------------
type memObjects struct {
mu sync.Mutex
name string
kept map[string][]byte
puts int
}
func newMemObjects() *memObjects { return &memObjects{name: "mail-test", kept: map[string][]byte{}} }
func (m *memObjects) Bucket() string { return m.name }
func (m *memObjects) Has(_ context.Context, key string) (bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
_, ok := m.kept[key]
return ok, nil
}
func (m *memObjects) Put(_ context.Context, key, _ string, data []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
m.kept[key] = append([]byte(nil), data...)
m.puts++
return nil
}
func (m *memObjects) Get(_ context.Context, key string) ([]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.kept[key]
if !ok {
return nil, fmt.Errorf("%s is not kept", key)
}
return b, nil
}
@@ -0,0 +1,104 @@
package main
import (
"context"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// The signature is AWS Signature Version 4 as S3 documents it: its own worked example, "GET Object", signs to the
// signature AWS publishes.
func TestTheSignatureIsTheOneS3Documents(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "https://examplebucket.s3.amazonaws.com/test.txt", nil)
req.Header.Set("Range", "bytes=0-9")
Sign(req, nil, "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "us-east-1", time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC))
want := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, " +
"SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
if got := req.Header.Get("Authorization"); got != want {
t.Fatalf("signed\n%s\nnot\n%s", got, want)
}
}
// An attachment is kept once by its SHA-256: the same file twice is one PUT, and it reads back.
func TestAnAttachmentIsKeptOnceByItsHash(t *testing.T) {
var mu sync.Mutex
kept := map[string][]byte{}
puts := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !strings.HasPrefix(r.Header.Get("Authorization"), "AWS4-HMAC-SHA256 Credential=mesh_novox_mail/") {
w.WriteHeader(http.StatusForbidden)
return
}
mu.Lock()
defer mu.Unlock()
switch r.Method {
case http.MethodHead:
if _, ok := kept[r.URL.Path]; !ok {
w.WriteHeader(http.StatusNotFound)
}
case http.MethodPut:
body, _ := io.ReadAll(r.Body)
kept[r.URL.Path] = body
puts++
case http.MethodGet:
b, ok := kept[r.URL.Path]
if !ok {
w.WriteHeader(http.StatusNotFound)
return
}
_, _ = w.Write(b)
}
}))
defer srv.Close()
dir := t.TempDir()
path := filepath.Join(dir, "store.json")
host, port, _ := strings.Cut(strings.TrimPrefix(srv.URL, "http://"), ":")
writeFile(t, path, `{"scheme": "http", "at": "`+host+`", "port": "`+port+`", "bucket": "mesh-novox-mail", "region": "eu-west",
"key": "mesh_novox_mail", "secret": "s3cr3t"}`)
facts, err := ReadStoreFacts(path)
if err != nil {
t.Fatal(err)
}
s3 := NewS3(facts)
ctx := context.Background()
a, err := Keep(ctx, s3, []byte("same file"), "text/plain", "a.txt")
if err != nil {
t.Fatal(err)
}
b, err := Keep(ctx, s3, []byte("same file"), "text/plain", "b.txt")
if err != nil {
t.Fatal(err)
}
if a.Key != b.Key || a.Bucket != "mesh-novox-mail" || puts != 1 || len(a.Key) != 64 {
t.Fatalf("%+v %+v, %d put(s)", a, b, puts)
}
if _, ok := kept["/mesh-novox-mail/"+a.Key]; !ok {
t.Fatalf("kept %v", kept)
}
got, err := s3.Get(ctx, a.Key)
if err != nil || string(got) != "same file" {
t.Fatalf("%q %v", got, err)
}
}
func TestAMessagesTextAndAttachmentsAreRead(t *testing.T) {
p, err := ParseMessage([]byte(strings.ReplaceAll(attachedMail("x"), "\n", "\r\n")))
if err != nil {
t.Fatal(err)
}
if strings.TrimSpace(p.Text) != "Please find the invoice attached." || len(p.Attachments) != 1 ||
p.Attachments[0].Name != "invoice.pdf" || p.Attachments[0].Type != "application/pdf" || string(p.Attachments[0].Data) != attachmentData {
t.Fatalf("%+v", p)
}
html := "From: a@b.c\r\nSubject: s\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><head><style>p{}</style></head><body><p>Hello &amp; welcome</p><script>alert(1)</script><p>Bye</p></body></html>"
p, err = ParseMessage([]byte(html))
if err != nil || p.Text != "Hello & welcome\nBye" {
t.Fatalf("%q %v", p.Text, err)
}
}
+504
View File
@@ -0,0 +1,504 @@
package main
// The index on the mesh's postgres (novox/hq ADR 0359 §3): the database the mesh provisions for this module through
// the postgres-database provision, its address in the file DATABASE_URL_FILE names. Its schema is numbered migrations
// in migrations/, applied by the preparation step (`mesh-mail prepare`, ADR 0135) and recorded in the database itself;
// a fresh database is given the consolidated schema, schema.sql, at once, with every migration recorded as applied.
// Nothing else changes the schema: the keeper, the watch and the tools refuse a schema that is behind, and say so.
import (
"context"
"crypto/sha256"
"embed"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path"
"regexp"
"sort"
"strconv"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrationFiles embed.FS
//go:embed schema.sql
var consolidated string
// Migration is one numbered change of the schema.
type Migration struct {
Number int
Name string
SQL string
Checksum string
}
var migrationFile = regexp.MustCompile(`^(\d{4})-([a-z0-9-]+)\.sql$`)
// Migrations are the migrations this binary carries, in order, numbered from 0001 without a gap.
func Migrations() ([]Migration, error) {
entries, err := fs.ReadDir(migrationFiles, "migrations")
if err != nil {
return nil, err
}
var out []Migration
for _, e := range entries {
m := migrationFile.FindStringSubmatch(e.Name())
if m == nil {
return nil, fmt.Errorf("%s is not a migration's name (NNNN-what-it-does.sql)", e.Name())
}
n, _ := strconv.Atoi(m[1])
body, err := fs.ReadFile(migrationFiles, path.Join("migrations", e.Name()))
if err != nil {
return nil, err
}
if strings.TrimSpace(string(body)) == "" {
return nil, fmt.Errorf("%s is empty", e.Name())
}
sum := sha256.Sum256(body)
out = append(out, Migration{Number: n, Name: m[2], SQL: string(body), Checksum: hex.EncodeToString(sum[:])})
}
sort.Slice(out, func(i, j int) bool { return out[i].Number < out[j].Number })
for i, m := range out {
if m.Number != i+1 {
return nil, fmt.Errorf("the migrations are not numbered 0001 on without a gap: %04d is the %d. one", m.Number, i+1)
}
}
return out, nil
}
const ledger = `create table if not exists migration (
number integer primary key,
name text not null,
checksum text not null,
applied timestamptz not null default now())`
// lockKey is the advisory lock every migration run takes, so two preparations never interleave.
const lockKey int64 = 4_250_359_001
// PgStore is the Store on postgres.
type PgStore struct{ pool *pgxpool.Pool }
// PgStoreFromEnv opens the index the mesh provisioned.
func PgStoreFromEnv(ctx context.Context) (*PgStore, error) {
file := os.Getenv("DATABASE_URL_FILE")
if file == "" {
return nil, errors.New("DATABASE_URL_FILE is not set: the index's database is a requirement the mesh resolves")
}
raw, err := os.ReadFile(file)
if err != nil {
return nil, fmt.Errorf("the database's address cannot be read: %w", err)
}
return OpenPgStore(ctx, strings.TrimSpace(string(raw)))
}
// OpenPgStore opens the index at an address.
func OpenPgStore(ctx context.Context, url string) (*PgStore, error) {
pool, err := pgxpool.New(ctx, url)
if err != nil {
return nil, err
}
return &PgStore{pool: pool}, nil
}
// Close closes the store.
func (s *PgStore) Close() { s.pool.Close() }
// Prepare brings the schema to what this version needs, and says what it did.
func (s *PgStore) Prepare(ctx context.Context) ([]string, error) {
migrations, err := Migrations()
if err != nil {
return nil, err
}
conn, err := s.pool.Acquire(ctx)
if err != nil {
return nil, err
}
defer conn.Release()
if _, err := conn.Exec(ctx, `select pg_advisory_lock($1)`, lockKey); err != nil {
return nil, fmt.Errorf("the migration lock cannot be taken: %w", err)
}
defer func() { _, _ = conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock($1)`, lockKey) }()
if _, err := conn.Exec(ctx, ledger); err != nil {
return nil, err
}
applied := map[int]string{}
rows, err := conn.Query(ctx, `select number, checksum from migration`)
if err != nil {
return nil, err
}
highest := 0
for rows.Next() {
var n int
var sum string
if err := rows.Scan(&n, &sum); err != nil {
rows.Close()
return nil, err
}
applied[n] = sum
highest = max(highest, n)
}
rows.Close()
if err := rows.Err(); err != nil {
return nil, err
}
record := func(tx pgx.Tx, m Migration) error {
_, err := tx.Exec(ctx, `insert into migration (number, name, checksum) values ($1, $2, $3)`, m.Number, m.Name, m.Checksum)
return err
}
if len(applied) == 0 {
var exists bool
if err := conn.QueryRow(ctx, `select to_regclass('message') is not null`).Scan(&exists); err != nil {
return nil, err
}
if exists {
return nil, errors.New("the database holds the index's tables and no record of a migration: its contents are not known, so nothing is changed")
}
tx, err := conn.Begin(ctx)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, consolidated); err != nil {
return nil, fmt.Errorf("the consolidated schema failed, and nothing of it was kept: %w", err)
}
for _, m := range migrations {
if err := record(tx, m); err != nil {
return nil, err
}
}
if err := tx.Commit(ctx); err != nil {
return nil, err
}
return []string{fmt.Sprintf("a fresh database: the consolidated schema, through migration %04d", len(migrations))}, nil
}
var problems []string
var pending []Migration
for _, m := range migrations {
sum, ran := applied[m.Number]
switch {
case ran && sum != m.Checksum:
problems = append(problems, fmt.Sprintf("migration %04d-%s ran, and its file has changed since: write a new one", m.Number, m.Name))
case !ran && m.Number < highest:
problems = append(problems, fmt.Sprintf("migration %04d-%s never ran, and %04d did: renumber it above %04d", m.Number, m.Name, highest, highest))
case !ran:
pending = append(pending, m)
}
}
if len(problems) > 0 {
return nil, errors.New("this database and these migrations disagree: " + strings.Join(problems, "; "))
}
var said []string
for _, m := range pending {
err := func() error {
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
defer func() { _ = tx.Rollback(context.WithoutCancel(ctx)) }()
if _, err := tx.Exec(ctx, m.SQL); err != nil {
return fmt.Errorf("migration %04d-%s failed, and nothing it did was kept: %w", m.Number, m.Name, err)
}
if err := record(tx, m); err != nil {
return err
}
return tx.Commit(ctx)
}()
if err != nil {
return said, err
}
said = append(said, fmt.Sprintf("migration %04d-%s applied", m.Number, m.Name))
}
if len(said) == 0 {
said = []string{fmt.Sprintf("the schema is at migration %04d, as this version needs", len(migrations))}
}
return said, nil
}
// Ready says why the index cannot serve this version, or nil.
func (s *PgStore) Ready(ctx context.Context) error {
migrations, err := Migrations()
if err != nil {
return err
}
var at int
err = s.pool.QueryRow(ctx, `select coalesce(max(number), 0) from migration`).Scan(&at)
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) && pgErr.Code == "42P01" {
at = 0
} else if err != nil {
return fmt.Errorf("the database cannot be reached: %w", err)
}
if at != len(migrations) {
return fmt.Errorf("the database's schema is at migration %04d and this version needs %04d: the preparation step "+
"(mesh-mail prepare) has not run", at, len(migrations))
}
return nil
}
func (s *PgStore) Folder(ctx context.Context, account, folder string) (Folder, bool, error) {
f := Folder{Account: account, Name: folder}
var validity int64
err := s.pool.QueryRow(ctx, `select uidvalidity, messages, synced from folder where account = $1 and name = $2`, account, folder).
Scan(&validity, &f.Messages, &f.Synced)
if errors.Is(err, pgx.ErrNoRows) {
return f, false, nil
}
f.UIDValidity = uint32(validity)
return f, err == nil, err
}
func (s *PgStore) SetFolder(ctx context.Context, f Folder) error {
_, err := s.pool.Exec(ctx, `insert into folder (account, name, uidvalidity, messages, synced) values ($1, $2, $3, $4, $5)
on conflict (account, name) do update set uidvalidity = excluded.uidvalidity, messages = excluded.messages, synced = excluded.synced`,
f.Account, f.Name, int64(f.UIDValidity), f.Messages, f.Synced)
return err
}
func (s *PgStore) DropFolder(ctx context.Context, account, folder string) error {
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `delete from message where account = $1 and folder = $2`, account, folder); err != nil {
return err
}
_, err := tx.Exec(ctx, `delete from folder where account = $1 and name = $2`, account, folder)
return err
})
}
func (s *PgStore) Flags(ctx context.Context, account, folder string) (map[uint32][]string, error) {
rows, err := s.pool.Query(ctx, `select uid, flags from message where account = $1 and folder = $2`, account, folder)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[uint32][]string{}
for rows.Next() {
var uid int64
var flags []string
if err := rows.Scan(&uid, &flags); err != nil {
return nil, err
}
out[uint32(uid)] = flags
}
return out, rows.Err()
}
func keepEvent(ctx context.Context, tx pgx.Tx, e *Event) error {
if e == nil {
return nil
}
body, err := json.Marshal(e.Body)
if err != nil {
return err
}
_, err = tx.Exec(ctx, `insert into outbox (key, body) values ($1, $2)`, e.Key, body)
return err
}
func (s *PgStore) Add(ctx context.Context, m Message, e *Event) error {
from, _ := json.Marshal(m.From)
to, _ := json.Marshal(m.To)
cc, _ := json.Marshal(orNone(m.Cc))
atts, _ := json.Marshal(m.Attachments)
flags := m.Flags
if flags == nil {
flags = []string{}
}
people := strings.Join([]string{peopleText(m.From), peopleText(m.To), peopleText(m.Cc)}, " ")
// An address is one word to the search, so its parts are kept beside it: "bob" finds bob@example.org.
people += " " + strings.NewReplacer("@", " ", ".", " ", ",", " ").Replace(people)
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `insert into message (account, folder, uid, uidvalidity, message_id, in_reply_to, date, sender,
recipients, copied, people, subject, size, flags, attachments, body)
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
on conflict (account, folder, uid) do update set uidvalidity = excluded.uidvalidity, message_id = excluded.message_id,
in_reply_to = excluded.in_reply_to, date = excluded.date, sender = excluded.sender, recipients = excluded.recipients,
copied = excluded.copied, people = excluded.people, subject = excluded.subject, size = excluded.size,
flags = excluded.flags, attachments = excluded.attachments, body = excluded.body`,
m.Account, m.Folder, int64(m.UID), int64(m.UIDValidity), m.MessageID, m.InReplyTo, m.Date, from, to, cc, people,
strings.ToValidUTF8(strings.ReplaceAll(m.Subject, "\x00", ""), ""), m.Size, flags, atts,
strings.ReplaceAll(m.Body, "\x00", "")); err != nil {
return err
}
return keepEvent(ctx, tx, e)
})
}
func orNone(ps []Person) []Person {
if ps == nil {
return []Person{}
}
return ps
}
func (s *PgStore) SetFlags(ctx context.Context, account, folder string, uid uint32, flags []string, e *Event) error {
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `update message set flags = $4 where account = $1 and folder = $2 and uid = $3`,
account, folder, int64(uid), flags); err != nil {
return err
}
return keepEvent(ctx, tx, e)
})
}
func (s *PgStore) Remove(ctx context.Context, account, folder string, uid uint32, e *Event) error {
return pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
if _, err := tx.Exec(ctx, `delete from message where account = $1 and folder = $2 and uid = $3`, account, folder, int64(uid)); err != nil {
return err
}
return keepEvent(ctx, tx, e)
})
}
const messageColumns = `account, folder, uid, uidvalidity, message_id, in_reply_to, date, sender, recipients, copied, subject,
size, flags, attachments`
func scanMessage(row pgx.Row, withBody bool) (Message, error) {
var m Message
var uid, validity int64
var from, to, cc, atts []byte
dest := []any{&m.Account, &m.Folder, &uid, &validity, &m.MessageID, &m.InReplyTo, &m.Date, &from, &to, &cc, &m.Subject,
&m.Size, &m.Flags, &atts}
if withBody {
dest = append(dest, &m.Body)
}
if err := row.Scan(dest...); err != nil {
return m, err
}
m.UID, m.UIDValidity, m.Date = uint32(uid), uint32(validity), m.Date.UTC()
for _, f := range []struct {
raw []byte
into any
}{{from, &m.From}, {to, &m.To}, {cc, &m.Cc}, {atts, &m.Attachments}} {
if err := json.Unmarshal(f.raw, f.into); err != nil {
return m, err
}
}
return m, nil
}
func (s *PgStore) Get(ctx context.Context, account, folder string, uid uint32) (Message, bool, error) {
m, err := scanMessage(s.pool.QueryRow(ctx, `select `+messageColumns+`, body from message where account = $1 and folder = $2 and uid = $3`,
account, folder, int64(uid)), true)
if errors.Is(err, pgx.ErrNoRows) {
return Message{}, false, nil
}
return m, err == nil, err
}
func (s *PgStore) Search(ctx context.Context, q Query) ([]Message, error) {
limit := q.Limit
if limit <= 0 {
limit = 20
}
rows, err := s.pool.Query(ctx, `select `+messageColumns+` from message
where account = any($1) and ($2 = '' or folder = $2) and search @@ plainto_tsquery('simple'::regconfig, $3)
order by date desc limit $4`, q.Accounts, q.Folder, q.Text, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Message
for rows.Next() {
m, err := scanMessage(rows, false)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
func (s *PgStore) Folders(ctx context.Context, account string) ([]Folder, error) {
rows, err := s.pool.Query(ctx, `select name, uidvalidity, messages, synced from folder where account = $1 order by name`, account)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Folder
for rows.Next() {
f := Folder{Account: account}
var validity int64
if err := rows.Scan(&f.Name, &validity, &f.Messages, &f.Synced); err != nil {
return nil, err
}
f.UIDValidity, f.Synced = uint32(validity), f.Synced.UTC()
out = append(out, f)
}
return out, rows.Err()
}
func (s *PgStore) Pending(ctx context.Context, limit int) ([]Event, error) {
rows, err := s.pool.Query(ctx, `select id, key, body from outbox order by id limit $1`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Event
for rows.Next() {
var e Event
var body []byte
if err := rows.Scan(&e.ID, &e.Key, &body); err != nil {
return nil, err
}
if err := json.Unmarshal(body, &e.Body); err != nil {
return nil, err
}
out = append(out, e)
}
return out, rows.Err()
}
func (s *PgStore) Said(ctx context.Context, ids []int64) error {
_, err := s.pool.Exec(ctx, `delete from outbox where id = any($1)`, ids)
return err
}
func (s *PgStore) SetState(ctx context.Context, st AccountState) error {
_, err := s.pool.Exec(ctx, `insert into account_state (account, state, why, heard, synced, said) values ($1, $2, $3, $4, $5, $6)
on conflict (account) do update set state = excluded.state, why = excluded.why, heard = excluded.heard,
synced = excluded.synced, said = excluded.said`,
st.Account, st.State, st.Why, nullTime(st.Heard), nullTime(st.Synced), st.Said)
return err
}
func nullTime(t time.Time) *time.Time {
if t.IsZero() {
return nil
}
return &t
}
func (s *PgStore) States(ctx context.Context) (map[string]AccountState, error) {
rows, err := s.pool.Query(ctx, `select account, state, why, heard, synced, said from account_state`)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]AccountState{}
for rows.Next() {
var st AccountState
var heard, synced *time.Time
if err := rows.Scan(&st.Account, &st.State, &st.Why, &heard, &synced, &st.Said); err != nil {
return nil, err
}
if heard != nil {
st.Heard = heard.UTC()
}
if synced != nil {
st.Synced = synced.UTC()
}
st.Said = st.Said.UTC()
out[st.Account] = st
}
return out, rows.Err()
}

Some files were not shown because too many files have changed in this diff Show More