Compare commits

...
61 Commits
Author SHA1 Message Date
mesh-tools 97a111e70a Merge pull request 'Hold the Approve ask until a review names the head (issue 1803, ADR 0496)' (#432) from fix/1803-hold-ask-without-review into main 2026-10-21 23:48:59 +00:00
mesh-tools eb4f16431c Merge pull request 'Leave the go.mod fixtures under testdata out of the build module's tests and digests (issue 1810)' (#431) from fix/1810-beside-sibling into main 2026-10-21 23:44:20 +00:00
jschoubben c4fd31aa7e Say a held group waits for a review and keep each member's status current (issue 1803, round 1)
mesh/merge-gate head c4fd31aa7e97 merged into trunk eb4f16431ce3 — pass: builds mesh-delivery-gate → novox; no bus step; every machine composes with the ch…
mesh/repo-check head c4fd31aa7e97 merged into trunk eb4f16431ce3 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/check checking c4fd31aa7e97 on 97a111e70abf
2026-10-22 01:37:49 +02:00
jschoubben 09d5360bce Follow the hold in the bus test: a head with no review is asked only once an edit names it (issue 1803)
mesh/merge-gate head 09d5360bce5a merged into trunk 4915f6985785 — pass: builds mesh-delivery-gate → novox; no bus step; every machine composes with the ch…
mesh/repo-check head 09d5360bce5a merged into trunk 4915f6985785 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/check failed: unit: clipmenu/cmd/clipmenu-tools.TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock: manifest_test.go:199: the launcher …
mesh/delivery superseded: a newer head of the same pull request
2026-10-22 01:27:18 +02:00
jschoubben b4d7a32abf Hold the Approve ask until a review names the head (issue 1803, ADR 0496)
Twice an unreviewed pull request was asked and merged on a warning line alone.
2026-10-22 01:25:15 +02:00
jschoubben 6aa4f135c9 Leave the go.mod fixtures under testdata out of the build module's tests and digests, so the checker stops running a fixture as a module (issue 1810)
mesh/check failed: unit: i3/cmd/i3-tools.TestTheMainFileAndEveryModulesContributionLoadTogether: manifest_test.go:247: expected the launcher, clipboar…
mesh/merge-gate head 6aa4f135c9f6 merged into trunk 4915f6985785 — pass: builds mesh-build-go → ace, g14, novox, shanks; no bus step; every machine compose…
mesh/repo-check head 6aa4f135c9f6 merged into trunk 4915f6985785 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-22 01:22:31 +02:00
mesh-tools 4915f69857 Merge pull request 'Add disk_folders to disk-load: which folders fill a disk (issue 1804)' (#430) from fix/1804-folder-sizes into main 2026-10-21 23:20:59 +00:00
jschoubben 843e408556 disk_folders: answer said as a list (issue 1804)
mesh/merge-gate head 843e40855684 merged into trunk 3bbae41037e3 — pass: builds disk-load → ace, g14, novox, shanks; no bus step; every machine composes wi…
mesh/repo-check head 843e40855684 merged into trunk 3bbae41037e3 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/check failed: unit: clipmenu/cmd/clipmenu-tools.TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock: manifest_test.go:199: the launcher …
mesh/delivery delivered
2026-10-22 01:13:35 +02:00
jschoubben 4ef2fa2dff disk_folders: bound the walk at 25 s, below the tool runner's 30 s, and say findings as lines (issue 1804)
mesh/merge-gate waiting in the build queue since 01:13
mesh/delivery superseded: a newer head of the same pull request
mesh/check failed: unit: diskload/cmd/disk-load: # diskload/cmd/disk-load [diskload/cmd/disk-load.test] | 4ef2fa2dff1c on 3bbae41037e3 | unit 4096 run…
2026-10-22 01:13:29 +02:00
jschoubben 8288042eb7 Add disk_folders to disk-load: no tool said which folders fill a disk (issue 1804)
mesh/merge-gate building on shanks since 01:09
mesh/check failed: unit: clipmenu/cmd/clipmenu-tools.TestTheLauncherWaitsQuietlyWhileAnotherClipmenudHoldsTheLock: manifest_test.go:199: the launcher …
mesh/delivery superseded: a newer head of the same pull request
2026-10-22 01:02:31 +02:00
mesh-tools 3bbae41037 Merge pull request 'Start the gitea bundle's binds without awaiting them, so its handshake is answered (issue 1793)' (#429) from fix/1793-bind-after-handshake into main 2026-10-21 22:59:00 +00:00
jschoubben 77f3963d83 Test that the gitea bundle's import resolves while its binds are refused (issue 1793)
mesh/merge-gate head 77f3963d8338 merged into trunk c770f7947d95 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 77f3963d8338 merged into trunk c770f7947d95 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/check failed: unit: fixture/beside.TestReadsTheSiblingBesideIt: beside_test.go:17: open /var/lib/mesh-build-go-work/mvipvmt100c902/beside/sibling…
2026-10-22 00:49:27 +02:00
jschoubben 49a28f8537 Start the gitea bundle's binds without awaiting them, so its handshake is answered while a refused bind is asked again (issue 1793)
mesh/check failed: unit: modules/gitea.test: [gitea] where the poll of merges last looked from could not be read, so none is kept — no runtime | 49a28…
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head 49a28f85378b merged into trunk c770f7947d95 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 49a28f85378b merged into trunk c770f7947d95 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
2026-10-22 00:45:54 +02:00
mesh-tools c770f7947d Merge pull request 'Bind the gitea bundle's event consumers again until they hold, so verdicts reach the forge (issue 1793)' (#428) from fix/1793-verdict-reaches-the-forge into main 2026-10-21 22:44:16 +00:00
mesh-tools 4b8e7f11cf Merge pull request 'Make docker_volumes_remove_anonymous answer on a machine with over a thousand volumes (issue 1796)' (#427) from fix/1796-sizes-on-a-large-set into main 2026-10-21 22:38:36 +00:00
jschoubben 0d2686b5eb Bind the gitea bundle's event consumers again until they hold, so a refusal at its start no longer ends it and loses verdicts (issue 1793)
mesh/merge-gate head 0d2686b5ebee merged into trunk 4b8e7f11cf53 — pass: builds gitea → novox; no bus step; every machine composes with the change as it di…
mesh/repo-check head 0d2686b5ebee merged into trunk 4b8e7f11cf53 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/check failed: unit: fixture/beside.TestReadsTheSiblingBesideIt: beside_test.go:17: open /var/lib/mesh-build-go-work/mvip9ircc11d3c/beside/sibling…
2026-10-22 00:37:30 +02:00
mesh-tools 839be0eb64 Merge pull request 'A delivery whose walk was folded into a batch follows that batch's walk and ends delivered (issue 1786)' (#426) from fix/1786-follow-taken-over-walk into main 2026-10-21 22:26:33 +00:00
jschoubben dc144a4e7d Count volumes a half-done batch already removed as removed, and test the batching (issue 1796)
mesh/merge-gate head dc144a4e7dd7 merged into trunk 839be0eb6444 — pass: builds docker → ace, g14, novox, shanks; no bus step; every machine composes with …
mesh/repo-check head dc144a4e7dd7 merged into trunk 839be0eb6444 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
mesh/check failed: unit: fixture/beside.TestReadsTheSiblingBesideIt: beside_test.go:17: open /var/lib/mesh-build-go-work/mvip2dsm29adef/beside/sibling…
2026-10-22 00:23:21 +02:00
jschoubben bfc42e4f4f Read volume sizes only when asked and remove in bounded batches, so docker_volumes_remove_anonymous answers on a machine with over a thousand volumes (issue 1796)
mesh/merge-gate building on shanks since 00:21
mesh/delivery superseded: a newer head of the same pull request
2026-10-22 00:20:32 +02:00
jschoubben c4176107f2 Keep a delivery superseded by its walk final, and heal it as delivered by the walk that took its walk over (issue 1786)
mesh/merge-gate head c4176107f287 merged into trunk a3794ea156dd — pass: builds mesh-delivery-gate → novox; no bus step; every machine composes with the ch…
mesh/repo-check head c4176107f287 merged into trunk a3794ea156dd — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivering: 0 machine step(s) passed
2026-10-22 00:01:22 +02:00
mesh-tools a3794ea156 Merge pull request 'Add a tool that removes leftover anonymous docker volumes (issue 1796)' (#425) from fix/1796-remove-anonymous-volumes into main 2026-10-21 21:54:26 +00:00
jschoubben e8cb2ee470 Wait for the batch that took a folded walk over, and let a delivery superseded by its walk follow that walk (issue 1786)
mesh/merge-gate building on ace since 23:57
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 23:53:52 +02:00
jschoubben 07b3c5e77a Report each anonymous volume's size and the space freed (issue 1796)
mesh/merge-gate head 07b3c5e77a19 merged into trunk f3100d111633 — pass: builds docker → ace, g14, novox, shanks; no bus step; every machine composes with …
mesh/repo-check head 07b3c5e77a19 merged into trunk f3100d111633 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-21 23:46:46 +02:00
mesh-tools f3100d1116 Merge pull request 'The checker runs beside today's check: build modules merge and place siblings; with automx, provisioner and gate fixes' (#423) from feat/the-checker into main 2026-10-21 21:46:17 +00:00
jschoubben 64a96a13f7 Add docker_volumes_remove_anonymous: remove leftover unmounted anonymous volumes (issue 1796)
mesh/merge-gate building on ace since 23:45
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 23:44:18 +02:00
mesh-tools f1fb73e12c Merge pull request 'Ask a group member's own repo-check when the group is held for it (issue 1792)' (#424) from fix/1792-recheck-stale-member into main 2026-10-21 21:38:09 +00:00
jschoubben d2faada61b Clear a member's own-ask mark when its own verdict arrives, so a later pending status is asked again (issue 1792)
mesh/merge-gate head d2faada61bf3 merged into trunk 379317589488 — pass: builds mesh-delivery-gate → novox; no bus step; every machine composes with the ch…
mesh/repo-check head d2faada61bf3 merged into trunk 379317589488 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-21 23:16:35 +02:00
jschoubben f436d1dace Ask a group member's own repo-check when the group is held before its Approve for it (issue 1792)
mesh/merge-gate waiting in the build queue since 23:13
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 23:11:55 +02:00
jschoubben 12ff1cfbc4 Merge remote-tracking branch 'origin/fix/1106-sdk-member-merges' into feat/the-checker
mesh/repo-check head 12ff1cfbc434 merged into trunk f1fb73e12cee beside mesh-controller#430@b9ec1919 — pass: its merge-check.sh passed, each part in its to…
mesh/merge-gate head 12ff1cfbc434 merged into trunk f1fb73e12cee — pass: group feat/the-checker composed: every machine composes with the change as it did …
mesh/delivery delivered
mesh/delivery-group group feat/the-checker delivered: every member is delivered
2026-10-21 21:58:44 +02:00
jschoubben 7bf0de0ff5 Move the socket-return fill tests (issue 1165) to the integration level: they make a real socket and watch its directory 2026-10-21 21:57:53 +02:00
jschoubben 59b13d091d Ask no check of a group member that waits on its composed check, and stop asking a merge refused the same way with no trunk move (issue 1106) 2026-10-21 21:57:32 +02:00
jschoubben ce72a366bf Merge remote-tracking branch 'origin/feat/r4d-automx-go' into feat/the-checker 2026-10-21 21:50:26 +02:00
jschoubben d9876e5296 Merge remote-tracking branch 'origin/fix/1165-fill-retry' into feat/the-checker
# Conflicts:
#	modules/mesh-provisioner-client/cmd/mesh-provisioner-client/client_test.go
2026-10-21 21:50:26 +02:00
jschoubben 62e67578eb Merge remote-tracking branch 'origin/fix/xterm-config-defaults' into feat/the-checker 2026-10-21 21:50:00 +02:00
jschoubben 1319432e55 Test that the integration level gets MESH_CHECK_BESIDE only when siblings were placed (issue 1677) 2026-10-21 21:49:37 +02:00
jschoubben 0cf7b2344f Write a moved contributor's declared defaults into the provisioner client tests' stand-in mesh-config (issue 1780) 2026-10-21 21:48:59 +02:00
jschoubben aa5d263c08 Test the returning socket by a real restart of the stand-in node-engine 2026-10-21 21:45:34 +02:00
jschoubben 769a8d6f6a Fill again what failed for want of the node-engine's write socket, when inotify says the socket is back (issue 1165) 2026-10-21 21:45:13 +02:00
jschoubben 34b310b9f8 Drop the unused imports from the Node fetch preview 2026-10-21 21:24:44 +02:00
jschoubben 33d8b71cf7 Drop the unused imports from the Node fetch preview 2026-10-21 21:24:33 +02:00
jschoubben aa0eaa6c52 Build the merge preview's module trees with mesh-sdk's ModuleTrees and re-pin mesh-sdk to 78364e31, so the two fetch steps share it (issues 1675, 1676) 2026-10-21 21:24:24 +02:00
jschoubben d5804927b3 Make the build modules' fetch merge the head onto a trunk commit and place siblings beside the tree, so the checker judges the merge and tests read their siblings (issues 1675, 1676, 1677) 2026-10-21 21:04:59 +02:00
mesh-tools 3793175894 Merge pull request 'Make minio's link-key secrets 40 characters and keep every mc argument's secret out of its errors (issue 1766)' (#422) from fix/minio-link-key-length into main 2026-10-21 19:00:13 +00:00
jschoubben b70f246622 Merge remote-tracking branch 'origin/main' into fix/minio-link-key-length
mesh/merge-gate head b70f24662298 merged into trunk cdbdb4747474 — pass: builds minio, sent nowhere; no bus step; 1 wait(s) for a person; every machine com…
mesh/repo-check head b70f24662298 merged into trunk cdbdb4747474 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
2026-10-21 20:52:18 +02:00
mesh-tools cdbdb47474 Merge pull request 'xterm: every resource a config value, today's values as defaults (issue 1656)' (#416) from fix/1656-xterm-config into main 2026-10-21 18:36:28 +00:00
jschoubben 689e61081d Merge remote-tracking branch 'origin/main' into fix/minio-link-key-length
mesh/merge-gate head 689e61081df9 merged into trunk cdbdb4747474 — pass: builds minio, sent nowhere; no bus step; 1 wait(s) for a person; every machine com…
mesh/repo-check head 689e61081df9 merged into trunk cdbdb4747474 — fail: its merge-check.sh failed: mesh-controller: 1 problem(s) in 139 manifest(s): modul…
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 20:22:24 +02:00
jschoubben c83f48b821 Make link-key secrets 40 characters and keep every mc argument's secret out of its errors (issue 1766) 2026-10-21 20:21:56 +02:00
mesh-tools db5e6c5b48 Merge pull request 'Name the wait in the Stop hook's suggested calls, route adopt's counter reset through fsys, and stop writing empty counter files' (#421) from fix/1749-1757-wait-list into main 2026-10-21 18:19:19 +00:00
jschoubben b9dcf19c3f Merge remote-tracking branch 'origin/main' into fix/1656-xterm-config
mesh/repo-check head b9dcf19c3f91 merged into trunk db5e6c5b48f4 beside mesh-controller#420@9d260412 — pass: its merge-check.sh passed, each part in its to…
mesh/merge-gate head b9dcf19c3f91 merged into trunk db5e6c5b48f4 — pass: group fix/1656-xterm-config composed: every machine composes with the change as it…
mesh/delivery failed: its walk failed, carrying its own commit: a gate on a first machine (what it carried put back), a build, a machine
mesh/delivery-group group fix/1656-xterm-config failed: a member failed
2026-10-21 19:43:29 +02:00
jochen e3a714d5f0 Stop writing an empty refusal-counter file on every stop of every session (issue 1749)
mesh/merge-gate head e3a714d5f0f0 merged into trunk c3564c1accf3 — pass: builds claude-code → ace, g14, novox, shanks; no bus step; every machine composes …
mesh/repo-check head e3a714d5f0f0 merged into trunk c3564c1accf3 — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
writeRefusalCounters wrote "{}" and, since it now routes through fsys
(this branch's earlier commit), created the waits folder on every single
stop, even for a session that never had a wait: one counter file per
session id, piling up on a node that uses the wait list at all. It now
removes the file instead of writing empty counters, a no-op when there
was never one to begin with, so a stop with nothing to record leaves
neither the file nor the folder; a refusal still writes it, and the
stop right after the wait is done (or let through) removes it again.
2026-10-21 19:41:05 +02:00
jochen 08cec8cb1c Route adopt's refusal-counter reset through fsys, and name the wait in the Stop hook's suggestions (issues 1749, 1757)
issue 1749: AdoptName reset both session ids' refusal-counter files with
os.WriteFile, bypassing fsys; on a node whose agent has its own account
(ADR 0266) the module runs as the operator's account and cannot write
into the agent's home directly, so the reset silently failed. writeRefusalCounters
now goes through fsys (MkdirAll then WriteFile), as every other write into
the agent's home does; it is also called from the Stop hook itself, which
already runs as the agent's own account, where fsys is its zero value and
behaves exactly as the plain write did before.

issue 1757: the Stop hook's suggested watched and said calls left out `by`,
which claude_code_wait requires on every op, so a session following the
suggestion word for word was refused. Both now carry "by" from the wait's
own By field (already kept per wait since issue 1629). The SessionStart
hook's suggested adopt call already carried by; added a test for it too,
alongside the Stop hook's two, each extracting the exact JSON the hook
prints and feeding it back to the tool's own dispatch to prove it is
accepted as printed.
2026-10-21 19:35:33 +02:00
jschoubben 70500c21a8 Run mailu's automx as a Go program mounted into the mirrored alpine image, answering as captured but naming the IMAP server (issues 1631, 1632, 1700, 1701, ADR 0480)
Python automx2, its database and its automx_rows tool go; the container keeps its name, network and port, gains an http health.
2026-10-21 19:29:05 +02:00
mesh-tools c3564c1acc Merge pull request 'A session's wait list and the plugin's Stop hook, so a written-down wait is never left unwatched (ADR 0489, ADR 0490)' (#420) from feat/wait-list into main 2026-10-21 17:18:48 +00:00
jschoubben e4cbffbaa9 Merge remote-tracking branch 'origin/main' into fix/1656-xterm-config
mesh/merge-gate out of date: judged against trunk 7a7a4034852a, the trunk is now c3564c1accf3; judged again on a recheck or when it is merged
mesh/repo-check out of date: judged against trunk 7a7a4034852a, the trunk is now c3564c1accf3; judged again on a recheck or when it is merged
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 19:16:12 +02:00
jochen 0f159a1662 Fix review round 2 of the wait list and its hooks (issues 1629, 1743, 1744, ADR 0490)
mesh/merge-gate head 0f159a166200 merged into trunk 7a7a4034852a — pass: builds claude-code → ace, g14, novox, shanks; no bus step; every machine composes …
mesh/repo-check head 0f159a166200 merged into trunk 7a7a4034852a — pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (go), merge-…
mesh/delivery delivered
The Stop hook treated a missing waits file exactly like one that could
not be read: a notice on every stop of every session with no waits —
the normal state, not an exceptional one. DecideStop now treats
os.ErrNotExist as the ordinary case of no open waits, silently; the
notice is still said for a file that exists but cannot be read or does
not parse.

ADR 0490 §1 keeps waits under a name, but the file a hook reads is kept
per session id: writing it from only the name an op just acted on meant
a second name bound to the same session id had its waits silently drop
out of enforcement the moment the first name was touched, and adopting
one name away emptied the id's file outright even when another name was
still bound to it. Every write now rewrites a session's file from the
union of every name currently bound to it (sessionWaitsUnion), adopt's
rewrite of the superseded id included.

Also: each atomic write gets its own temp file name, so two writes to
the same path can never interleave on one; adopt clears the refusal
counter of both the superseded and the adopting session, so a fresh
three-strikes count starts under the new binding; and list is read-only
again — it no longer claims an unbound name or refuses a session that
is not (yet) the one bound to it, and instead says who currently holds
the name.
2026-10-21 18:43:03 +02:00
jochen f0beea153e Key a wait list by name, not session id, so it survives a /clear (issues 1629, 1744, ADR 0490)
Claude Code gives a session a new session_id at /clear, with nothing
naming the earlier one, so keying claude_code_wait's state by session id
lost a session's waits the moment it cleared. ADR 0490 changes ADR 0489
§1: waits are now kept under the caller's name and node (by, who@node);
every op also takes session, the caller's current id, and the module
keeps a binding of the name to the session that last called for it. An
unbound name is claimed by the first call for it; a session whose own
id no longer matches the binding is refused on every op, naming the
session that holds it, until it calls the new op adopt, which always
takes the name over at once (no bound protects a recent call) and tells
the earlier session so on its own next call.

The mirrored file a hook reads stays keyed by session id (a hook only
ever has that), holding whichever name's waits are currently bound to
it; adopting rewrites both the old id's file (emptied) and the new id's
(filled) in the same call. A second mirrored file, the node-wide index,
lists every name on the node with an open wait; the SessionStart hook
now also reads it, on every start, and prints the exact adopt call for
each name, so a session with a brand new id can find and reclaim its
own waits without calling back into the module.
2026-10-21 18:32:50 +02:00
jochen 02a8837348 Fix review round 1 of the wait list and its hooks (issues 1629, 1743, 1744)
A session id reaching the Stop and SessionStart hooks was only checked
for blank, so "../../x" or "a/b" could read or write outside the waits
directory; both hooks now hold every session id to waitSessionID before
building any path, and allow (saying why) rather than fail on a bad one.
claude_code_wait's list/watched/done/said held only add to that same
check, letting a malformed id through as an ordinary, always-empty key;
they all refuse it now.

The Stop hook's three-strikes cap could be bypassed forever if its
counter file could not be written on a refusal: it now allows instead,
saying so, so a wait can never hold a turn open indefinitely. Its advice
offered mesh_wait on a wait's event family unconditionally, ignoring
that the session's tool runner may not be granted it (ADR 0367 §2); it
now offers mesh_wait only "if this session's tool runner is granted it,
otherwise start a watcher", and drops the family entirely when a wait
has none.

Both hooks' fail-open and let-through notices were said only on stderr,
which Claude Code does not show the operator on an allowed (exit 0)
outcome; they now also print the same words as a "systemMessage" JSON
object on stdout, which Claude Code does show.
2026-10-21 18:23:42 +02:00
jochen 0fa427dad3 Add the plugin's Stop hook so a written-down wait is never left unwatched (issue 1743)
A session could end its turn while a wait it wrote down with
claude_code_wait had nothing following it. The plugin's Stop hook
(claude-code stop <waits-dir>) refuses the stop while a wait is within
its bound and unwatched, or past its bound and not said, reading the
waits file directly with no network and a 5s bound; it fails open on a
missing or malformed file or a panic, and lets a wait through after
three refusals in a row rather than holding the session forever.
Renames TestThePluginsOnlyHookIsTheGuard to cover the session-start and
Stop hooks too.
2026-10-21 18:08:13 +02:00
jochen 37628f4bce Add the plugin's SessionStart hook so a session learns its own id (issue 1744)
A session is never told its session id by any tool, only by a hook's
stdin, so it cannot name itself to claude_code_wait. The plugin's
SessionStart hook (claude-code session-start <waits-dir>) prints the id
and how to give it, plus the session's still-open waits after a /clear,
reading the waits file claude_code_wait mirrors directly rather than
calling back into this module. Carries the Stop hook's command and the
GuardFiles.WaitsDir plumbing MarketplaceWith needs for both (issue 1743
wires the Stop hook's dispatch next).
2026-10-21 18:07:36 +02:00
jochen 56c68be7fa Keep a session's wait list in the claude-code module (issue 1629)
A session can end its turn while an Approve it expects is still open and
nothing then watches it. claude_code_wait (add/watched/done/said/list)
keeps each session's waits in the module's waits state, keyed by its
session id and node, and mirrors them whole and atomically to a file
under the agent's home that the plugin's coming hooks (1743, 1744) will
read without calling back into this module.
2026-10-21 17:59:10 +02:00
jschoubben f178aab641 Make every xterm resource a config value with today's value as default; tools read the rendered resources (issue 1656)
mesh/delivery-group group fix/1656-xterm-config rejected: a member's own check failed
mesh/merge-gate waits for its group's composed check
mesh/repo-check head f178aab641dc merged into trunk 3259765d46bc beside mesh-controller#420@69a67876 — fail: its merge-check.sh failed: files asking for a …
mesh/delivery superseded: a newer head of the same pull request
2026-10-21 12:51:40 +02:00
114 changed files with 6977 additions and 728 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
59
58
The long-running resources of this catalogue that do not say how they are ready (`health`, novox/hq ADR 0240
rule 8), as `module check` counts them. It may only go down: merge-check.sh fails a change that raises it, and
one that lowers it writes the new number on the first line. From 2026-11-18, or once it is 0, a long-running
+28 -3
View File
@@ -476,10 +476,13 @@ type PluginFile struct {
// what it is given. Outside hooks/, so no registered hook's name can be it.
const GuardDir = "guard"
// GuardFiles are the guard as the plugin carries it: the binary to copy and the data it judges with.
// GuardFiles are the guard as the plugin carries it: the binary to copy, what it judges with, and — the same
// binary again — the waits directory the SessionStart and Stop hooks are given (novox/hq ADR 0489, issues
// 1744, 1743). Empty WaitsDir places no such hook.
type GuardFiles struct {
Exe string
Data GuardData
Exe string
Data GuardData
WaitsDir string
}
// GuardMatcher is what the guard is asked about: every tool. Its own rules judge the shell and the agent's file
@@ -495,6 +498,20 @@ func GuardCommand() string {
return `"` + filepath.Join(dir, "guard") + `" guard "` + filepath.Join(dir, "guard.json") + `"`
}
// SessionStartCommand is the plugin's SessionStart hook: the guard's own binary, told the session's open
// waits live under waitsDir (novox/hq ADR 0489 §3, issue 1744).
func SessionStartCommand(waitsDir string) string {
dir := filepath.Join(ManagedDir, MarketplaceDir, "plugins", Plugin, GuardDir)
return `"` + filepath.Join(dir, "guard") + `" session-start "` + waitsDir + `"`
}
// StopCommand is the plugin's Stop hook: the guard's own binary, refusing to end the turn while a wait under
// waitsDir is open and unwatched, or past its bound and untold (novox/hq ADR 0489 §3-§6, issue 1743).
func StopCommand(waitsDir string) string {
dir := filepath.Join(ManagedDir, MarketplaceDir, "plugins", Plugin, GuardDir)
return `"` + filepath.Join(dir, "guard") + `" stop "` + waitsDir + `"`
}
// Marketplace is the marketplace directory's whole content, by path inside it. A node item of a name laid
// over a mesh item of the same kind and name: the node's wins.
func Marketplace(c Config) map[string]PluginFile { return MarketplaceWith(c, nil) }
@@ -533,6 +550,14 @@ func MarketplaceWith(c Config, guard *GuardFiles) map[string]PluginFile {
out[root+GuardDir+"/guard.json"] = PluginFile{Content: jsonFile(guard.Data)}
hooks["PreToolUse"] = append(hooks["PreToolUse"], map[string]any{"matcher": GuardMatcher,
"hooks": []any{map[string]any{"type": "command", "command": GuardCommand(), "timeout": 15}}})
if guard.WaitsDir != "" {
// A session's wait list (novox/hq ADR 0489): the SessionStart hook names the session's id and its open
// waits (issue 1744); the Stop hook keeps the turn open while one is unwatched or untold (issue 1743).
hooks["SessionStart"] = append(hooks["SessionStart"], map[string]any{"matcher": "startup|resume|clear|compact",
"hooks": []any{map[string]any{"type": "command", "command": SessionStartCommand(guard.WaitsDir), "timeout": 10}}})
hooks["Stop"] = append(hooks["Stop"], map[string]any{
"hooks": []any{map[string]any{"type": "command", "command": StopCommand(guard.WaitsDir), "timeout": 5}}})
}
}
for _, k := range keys {
it := chosen[k]
@@ -0,0 +1,139 @@
package main
// Every call a hook prints for a session to copy must be accepted by claude_code_wait's own argument check
// (issue 1757): the Stop hook's suggested watched and said calls, and the SessionStart hook's suggested
// adopt call.
import (
"encoding/json"
"strings"
"testing"
"time"
)
// suggestedCall extracts the one JSON object embedded in a hook's message, right after "claude_code_wait "
// — what a session is expected to copy verbatim — and parses it into the arguments a tool call takes.
func suggestedCall(t *testing.T, msg string) map[string]any {
t.Helper()
i := strings.Index(msg, "claude_code_wait ")
if i < 0 {
t.Fatalf("no suggested call in: %s", msg)
}
rest := msg[i+len("claude_code_wait "):]
start := strings.Index(rest, "{")
if start < 0 {
t.Fatalf("no JSON object in: %s", msg)
}
depth, end := 0, -1
for j, c := range rest[start:] {
switch c {
case '{':
depth++
case '}':
depth--
if depth == 0 {
end = start + j + 1
}
}
if end >= 0 {
break
}
}
if end < 0 {
t.Fatalf("unbalanced JSON object in: %s", msg)
}
var args map[string]any
if err := json.Unmarshal([]byte(rest[start:end]), &args); err != nil {
t.Fatalf("the suggested call is not valid JSON: %v (%s)", err, rest[start:end])
}
return args
}
// Issue 1757: the Stop hook's suggested watched call carries `by`, and the tool accepts it as printed.
func TestTheStopHooksSuggestedWatchedCallIsAcceptedByTheTool(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
t.Cleanup(func() { waiting = nil })
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "verify-0489-test", "by": "fix-1757@g14",
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
if isErr {
t.Fatalf("add: %s", text)
}
f, err := readWaitsFile(waitsFilePath(p, "verify-0489-test"))
if err != nil || len(f.Waits) != 1 {
t.Fatalf("%+v %v", f, err)
}
_, reason := waitBlocksStop("verify-0489-test", f.Waits[0], t0.Add(time.Minute))
if !strings.Contains(reason, "\"by\":\"fix-1757\"") {
t.Fatalf("the suggested call does not carry by: %s", reason)
}
args := suggestedCall(t, reason)
text, isErr = s.call(t, "claude_code_wait", args)
if isErr {
t.Fatalf("the suggested watched call, as printed, was refused: %s", text)
}
}
// Issue 1757: the Stop hook's suggested said call (past its bound) carries `by` too.
func TestTheStopHooksSuggestedSaidCallIsAcceptedByTheTool(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
clock := t0
waiting = &Waiting{Store: store, Now: func() time.Time { return clock }}
t.Cleanup(func() { waiting = nil })
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "verify-0489-test", "by": "fix-1757@g14",
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 1})
if isErr {
t.Fatalf("add: %s", text)
}
clock = t0.Add(2 * time.Minute) // past its 1-minute bound, for the live tool call too
f, err := readWaitsFile(waitsFilePath(p, "verify-0489-test"))
if err != nil || len(f.Waits) != 1 {
t.Fatalf("%+v %v", f, err)
}
_, reason := waitBlocksStop("verify-0489-test", f.Waits[0], clock)
if !strings.Contains(reason, "\"by\":\"fix-1757\"") {
t.Fatalf("the suggested call does not carry by: %s", reason)
}
args := suggestedCall(t, reason)
text, isErr = s.call(t, "claude_code_wait", args)
if isErr {
t.Fatalf("the suggested said call, as printed, was refused: %s", text)
}
}
// Issue 1757: the SessionStart hook's suggested adopt call already carries `by` — checked, as asked, so a
// regression here is caught the same way as the Stop hook's.
func TestTheSessionStartHooksSuggestedAdoptCallIsAcceptedByTheTool(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
t.Cleanup(func() { waiting = nil })
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "sess-old", "by": "fix-1757@g14",
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
if isErr {
t.Fatalf("add: %s", text)
}
msg := SessionStartMessage("sess-new", WaitsDirPath(p), t0.Add(time.Minute))
if !strings.Contains(msg, "\"by\":\"fix-1757@g14\"") {
t.Fatalf("the suggested adopt call does not carry by: %s", msg)
}
args := suggestedCall(t, msg)
text, isErr = s.call(t, "claude_code_wait", args)
if isErr {
t.Fatalf("the suggested adopt call, as printed, was refused: %s", text)
}
}
@@ -0,0 +1,23 @@
package main
// Shared by the plugin's hooks (novox/hq ADR 0489 §5): a notice on exit 0 — the stop hook failing open or
// letting a wait through, the session-start hook unable to use a session id — reaches the operator's screen,
// not only the log. Claude Code shows a hook's top-level "systemMessage" field to the person; stderr alone,
// on an allowed (exit 0) outcome, is never shown there.
import (
"encoding/json"
"fmt"
"io"
)
// writeSystemMessage prints message as the one JSON object Claude Code shows the operator, falling back to
// the plain words on the one error that cannot happen (a string always marshals).
func writeSystemMessage(w io.Writer, message string) {
raw, err := json.Marshal(map[string]string{"systemMessage": message})
if err != nil {
fmt.Fprintln(w, message)
return
}
fmt.Fprintln(w, string(raw))
}
@@ -388,6 +388,7 @@ func tools(p Paths, servers ServerState, view *ServerView, config ConfigState, c
Description: "Read-only: what this node carries of the agent's memory between nodes (novox/hq ADR 0343) — the home kind it carries for (the operator's own homes, or an agent account's), each project's memory files with their fingerprint and the node and time of their last version, the files not carried and why, the conflicts still open (a file changed on two nodes, kept twice in one file, the newer first), and how many replaced or removed versions are kept. Fingerprints only, never the text; it changes nothing.",
Run: func(map[string]any) (any, error) { return memoryCarrier.Status(), nil }})
out = append(out, proposalTools(p)...)
out = append(out, waitTools(p)...)
judge := NewJudge(p)
out = append(out, judgeTool(judge), judgedTool(judge))
return append(out, configTools(p, config, configView)...)
@@ -416,6 +417,15 @@ func main() {
if len(os.Args) == 3 && os.Args[1] == "guard" {
os.Exit(runGuard(os.Args[2], os.Stdin, os.Stderr))
}
// The plugin's SessionStart hook (novox/hq ADR 0489 §1, issue 1744): names the session's id and its open waits.
if len(os.Args) == 3 && os.Args[1] == "session-start" {
os.Exit(runSessionStart(os.Args[2], os.Stdin, os.Stdout))
}
// The plugin's Stop hook (novox/hq ADR 0489 §3-§6, issue 1743): keeps the turn open while a wait is unwatched
// or untold.
if len(os.Args) == 3 && os.Args[1] == "stop" {
os.Exit(runStop(os.Args[2], os.Stdin, os.Stdout, os.Stderr))
}
// The helper that touches the agent's home as the agent's account (novox/hq ADR 0266), started by this
// module through sudo: one answer per request on stdin, until it ends.
if len(os.Args) == 2 && os.Args[1] == HelperArg {
@@ -463,6 +473,7 @@ func main() {
proposing = &Proposing{Store: stateOf{stdio.State(ProposalsState)}, Bus: sdkAskBus{}, Record: recordWarrant,
Now: time.Now, Write: writeManaged}
go proposing.watch(p)
waiting = &Waiting{Store: keptWaits{stdio.State(WaitsState)}, Now: time.Now}
memoryCarrier = NewMemoryCarrier(p, memoryStateOf{stdio.State(MemoryState)}, notifyOperator)
go memoryCarrier.Run(func(handle func(key, op string, value json.RawMessage, revision uint64) error) error {
return stdio.State(MemoryState).Watch("", func(c stdio.StateChange) error {
+1 -1
View File
@@ -278,7 +278,7 @@ func guardFiles(p Paths, m *MeshTools) *GuardFiles {
if resolved, err := filepath.EvalSymlinks(exe); err == nil {
exe = resolved
}
return &GuardFiles{Exe: exe, Data: GuardDataOf(p, m)}
return &GuardFiles{Exe: exe, Data: GuardDataOf(p, m), WaitsDir: WaitsDirPath(p)}
}
// MeshToolsEvery is how often the controller is asked again.
@@ -0,0 +1,86 @@
package main
// The plugin's SessionStart hook (novox/hq ADR 0489 §1, ADR 0490 §2, issue 1744): `claude-code session-start
// <waits-dir>`, run by the agent's own account on every start, resume, /clear and /compact. The tool runner
// never tells a session its own id (only a hook's stdin carries session_id), so a session cannot name itself
// to claude_code_wait until this hook has told it. It also lists this session id's own still-open waits (if
// any name is already bound to it) and, always, every name on this node that still holds an open wait, so a
// session with a brand new id (a /clear gives one) can see a name it may want to adopt.
//
// It reads the session's mirrored waits file and the node-wide index directly — the same local reads the
// Stop hook and claude_code_wait make (readWaitsFile, readNodeIndex, waits.go) — never calling back into
// this module: this hook runs on every start, and must not wait on the bus to do it. A missing or malformed
// file is not an error here: there is simply nothing open to report.
import (
"encoding/json"
"fmt"
"io"
"strings"
"time"
)
// SessionStartInput is what the hook is given on stdin.
type SessionStartInput struct {
SessionID string `json:"session_id"`
}
// SessionStartMessage is the line the hook prints: the session's id and how to give it to claude_code_wait,
// then each still-open wait already bound to this session id (1744/AC1, AC2), then every name on this node
// that still holds an open wait, with how to adopt it (ADR 0490 §2). Pure of IO beyond the two file reads.
func SessionStartMessage(sessionID, waitsDir string, now time.Time) string {
var b strings.Builder
fmt.Fprintf(&b, "This session's id is %s; give it as `session` to claude_code_wait.\n", sessionID)
if f, err := readWaitsFile(waitsFileIn(waitsDir, sessionID)); err == nil && len(f.Waits) > 0 {
b.WriteString("Waits already bound to this session's own id:\n")
for _, w := range f.Waits {
age := now.Sub(w.At).Round(time.Second)
state := "no watcher yet"
if w.Watcher != "" {
state = "watched"
}
if w.PastBound(now) {
state += ", past its bound"
if w.Said {
state += ", said"
}
}
fmt.Fprintf(&b, " %s — %s (%s ago, %s)\n", w.ID, w.What, age, state)
}
}
if idx, err := readNodeIndex(nodeIndexPathIn(waitsDir)); err == nil && len(idx.Names) > 0 {
b.WriteString("Open waits on this node, by name:\n")
for _, n := range idx.Names {
fmt.Fprintf(&b, " %s: %d open wait(s), last called %s. If this is you: claude_code_wait "+
"{\"op\":\"adopt\",\"by\":%q,\"session\":%q}\n", n.Name, n.Open, n.LastCall.Round(time.Second).Format(time.RFC3339),
n.Name+"@"+idx.Node, sessionID)
}
}
return b.String()
}
// runSessionStart is the hook: the session's id and its open waits out, exit 0 always (1744/AC3).
func runSessionStart(waitsDir string, stdin io.Reader, stdout io.Writer) (code int) {
defer func() {
if recover() != nil {
code = 0 // never fails the session's start
}
}()
var in SessionStartInput
if err := json.NewDecoder(stdin).Decode(&in); err != nil {
return 0
}
sessionID := strings.TrimSpace(in.SessionID)
if sessionID == "" {
return 0
}
// Review round 1, item 1: a session id is never used to build a path until it is checked against
// waitSessionID — no "/" in it reaches the filesystem, so it can never name a file outside waitsDir.
if !waitSessionID.MatchString(sessionID) {
writeSystemMessage(stdout, fmt.Sprintf("this session's id (%q) could not be used to find its waits "+
"(letters, digits, '.', '_' and '-' only)", sessionID))
return 0
}
fmt.Fprint(stdout, SessionStartMessage(sessionID, waitsDir, time.Now()))
return 0
}
@@ -0,0 +1,151 @@
package main
// The plugin's SessionStart hook (novox/hq ADR 0489 §1, issue 1744).
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
// 1744/AC1: the hook prints the session's id and how to give it to claude_code_wait.
func TestSessionStartPrintsTheSessionsIDAndHowToGiveIt(t *testing.T) {
waitsDir := t.TempDir()
stdin := strings.NewReader(`{"session_id":"sess-9","hook_event_name":"SessionStart","source":"startup"}`)
var out bytes.Buffer
if code := runSessionStart(waitsDir, stdin, &out); code != 0 {
t.Fatalf("code %d, stdout %s", code, out.String())
}
if !strings.Contains(out.String(), "sess-9") || !strings.Contains(out.String(), "claude_code_wait") {
t.Fatalf("stdout does not say how to name the session: %s", out.String())
}
}
// 1744/AC2: when the waits file holds open waits, it prints each (what, age, past bound or not).
func TestSessionStartListsOpenWaitsAfterAClear(t *testing.T) {
waitsDir := t.TempDir()
at := time.Date(2026, 10, 21, 10, 0, 0, 0, time.UTC)
f := WaitsFile{Session: "sess-10", At: at, Waits: []Wait{
{ID: "w-open", What: "the operator's Approve", Bound: 10, At: at},
{ID: "w-past", What: "a delivery to finish", Bound: 1, At: at},
}}
writeWaitsFixture(t, filepath.Join(waitsDir, "sess-10.json"), f)
now := at.Add(5 * time.Minute)
msg := SessionStartMessage("sess-10", waitsDir, now)
if !strings.Contains(msg, "w-open") || !strings.Contains(msg, "the operator's Approve") || !strings.Contains(msg, "no watcher") {
t.Fatalf("the open, unwatched wait is not described:\n%s", msg)
}
if !strings.Contains(msg, "w-past") || !strings.Contains(msg, "past its bound") {
t.Fatalf("the past-bound wait is not described:\n%s", msg)
}
if !strings.Contains(msg, "5m0s ago") {
t.Fatalf("the age is not said:\n%s", msg)
}
}
// 1744/AC3: a missing or malformed file prints only the id line, and the hook never fails the session start.
func TestSessionStartNeverFailsOnAMissingOrBadFile(t *testing.T) {
waitsDir := t.TempDir()
if msg := SessionStartMessage("sess-11", waitsDir, time.Now()); !strings.Contains(msg, "sess-11") || strings.Contains(msg, "open") {
t.Fatalf("a missing file printed more than the id line: %q", msg)
}
writeFile(t, filepath.Join(waitsDir, "sess-12.json"), "{not json")
if msg := SessionStartMessage("sess-12", waitsDir, time.Now()); !strings.Contains(msg, "sess-12") || strings.Contains(msg, "open") {
t.Fatalf("a malformed file printed more than the id line: %q", msg)
}
var out bytes.Buffer
if code := runSessionStart(waitsDir, strings.NewReader("not json at all"), &out); code != 0 {
t.Fatalf("unreadable stdin failed the session start: code %d", code)
}
var out2 bytes.Buffer
if code := runSessionStart(waitsDir, panicReader{}, &out2); code != 0 {
t.Fatalf("a panic was not recovered: code %d", code)
}
}
// panicReader panics on Read, as a stuck pipe or a bug reading stdin might.
type panicReader struct{}
func (panicReader) Read([]byte) (int, error) { panic("boom") }
// Review round 1, item 1 (1743/1744, ADR 0489 §5): a session id that is not a plain one is never used to
// build a file path — reading outside the waits dir never happens — and the hook prints only a line saying
// the id could not be used.
func TestSessionStartRejectsAnUnsafeSessionID(t *testing.T) {
root := t.TempDir()
waitsDir := filepath.Join(root, "agent", "waits")
if err := os.MkdirAll(waitsDir, 0o700); err != nil {
t.Fatal(err)
}
// What "../../x" resolves to from waitsDir: root/x.json, outside waitsDir, with content that must never be
// read through it.
outside := filepath.Join(waitsDir, "../../x.json")
writeWaitsFixture(t, outside, WaitsFile{Session: "x", At: time.Now(),
Waits: []Wait{{ID: "w-secret", What: "SECRET-DATA", Bound: 10, At: time.Now()}}})
t.Cleanup(func() { _ = os.Remove(outside) })
for _, id := range []string{"../../x", "a/b"} {
var out bytes.Buffer
stdin := strings.NewReader(fmt.Sprintf(`{"session_id":%q}`, id))
if code := runSessionStart(waitsDir, stdin, &out); code != 0 {
t.Fatalf("id %q: code %d", id, code)
}
if strings.Contains(out.String(), "SECRET-DATA") {
t.Fatalf("id %q: read a file outside the waits dir: %s", id, out.String())
}
if !strings.Contains(out.String(), "could not be used") {
t.Fatalf("id %q: did not say the id could not be used: %s", id, out.String())
}
// Item (a): the notice reaches the screen as the one JSON object Claude Code shows.
if !strings.Contains(out.String(), "systemMessage") {
t.Fatalf("id %q: the notice is not a systemMessage: %s", id, out.String())
}
}
var out bytes.Buffer
if code := runSessionStart(waitsDir, strings.NewReader(`{"session_id":""}`), &out); code != 0 || out.Len() != 0 {
t.Fatalf("blank id: code %d, out %q", code, out.String())
}
}
// 1744/AC4 is checked together with 1743/AC7 in terminal_only_test.go's
// TestThePluginsOnlyHooksAreTheGuardTheSessionStartHookAndTheStopHook.
// ADR 0490 §2: every session start lists every name on this node with an open wait, from the node-wide
// index, with the exact adopt call to make if it is yours — read directly, never by calling back into this
// module.
func TestSessionStartListsEveryNameOnTheNodeWithOpenWaits(t *testing.T) {
waitsDir := t.TempDir()
lastCall := time.Date(2026, 10, 21, 9, 55, 0, 0, time.UTC)
idx := NodeIndex{Node: "g14", At: lastCall, Names: []NodeIndexEntry{
{Name: "fix-1629", Open: 2, CurrentSession: "sess-old", LastCall: lastCall},
}}
raw, err := indented(idx)
if err != nil {
t.Fatal(err)
}
writeFile(t, filepath.Join(waitsDir, "_index.json"), string(raw))
msg := SessionStartMessage("sess-brand-new", waitsDir, lastCall.Add(5*time.Minute))
if !strings.Contains(msg, "fix-1629") || !strings.Contains(msg, "2 open wait") {
t.Fatalf("the name and its count are not listed:\n%s", msg)
}
want := `claude_code_wait {"op":"adopt","by":"fix-1629@g14","session":"sess-brand-new"}`
if !strings.Contains(msg, want) {
t.Fatalf("the exact adopt call is not given:\n%s\nwant it to contain:\n%s", msg, want)
}
}
func writeWaitsFixture(t *testing.T, path string, f WaitsFile) {
t.Helper()
raw, err := indented(f)
if err != nil {
t.Fatal(err)
}
writeFile(t, path, string(raw))
}
+195
View File
@@ -0,0 +1,195 @@
package main
// The plugin's Stop hook (novox/hq ADR 0489 §3-§6, issue 1743): `claude-code stop <waits-dir>`, run by the
// agent's own account before a session's turn is let end. It reads the session's mirrored waits file
// directly (readWaitsFile, waits.go) — never the bus, never calling back into this module — so the decision
// is local, under 5 s, and needs no network.
//
// Within its bound, a wait needs a watcher: refused while one is open and nothing follows it yet. Past its
// bound, a wait needs a word to the operator, watched or not: refused until it is marked said. Refused at
// most three times running for the same wait (a counter file beside the waits file, reset once the blocking
// wait changes or clears); the fourth time it is let through, saying the wait was left unwatched. A missing,
// unreadable or malformed waits file, or a panic, always allows.
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
)
// StopInput is what the hook is given on stdin.
type StopInput struct {
SessionID string `json:"session_id"`
TranscriptPath string `json:"transcript_path"`
StopHookActive bool `json:"stop_hook_active"`
}
// MostStopRefusals is how many times in a row the Stop hook refuses for the same wait before it lets the
// turn end anyway (1743/AC5).
const MostStopRefusals = 3
// StopDecision is the Stop hook's verdict, and the refusal counters it leaves behind for the next call.
type StopDecision struct {
Refuse bool
Reason string
Counters map[string]int
}
// waitBlocksStop says whether w, on its own, keeps the turn from ending, and why (ADR 0489 §3-§4).
func waitBlocksStop(session string, w Wait, now time.Time) (bool, string) {
if w.Said {
return false, ""
}
if !w.PastBound(now) {
if w.Watcher != "" {
return false, ""
}
return true, fmt.Sprintf("wait %s (%s) is open and nothing follows it yet. Read where it stands with %s, then "+
"%s, and mark it watched: claude_code_wait "+
"{\"op\":\"watched\",\"session\":%q,\"by\":%q,\"id\":%q,\"watcher\":\"...\"}.", w.ID, w.What, w.Read.Address,
watcherAdvice(w), session, w.By, w.ID)
}
return true, fmt.Sprintf("wait %s (%s) is past its bound and the operator has not been told. Tell the operator what "+
"waits, since when, and what you are doing about it, then mark it: claude_code_wait "+
"{\"op\":\"said\",\"session\":%q,\"by\":%q,\"id\":%q}.", w.ID, w.What, session, w.By, w.ID)
}
// watcherAdvice is what a session is told to do to follow a wait (ADR 0489 §3, review round 1 item 3): a
// family is a candidate for mesh_wait only when the tool runner is granted it (ADR 0367 §2), which this hook
// cannot know, so it is offered conditionally; a watcher that rereads the read address always works.
func watcherAdvice(w Wait) string {
if w.Family != "" {
return fmt.Sprintf("mesh_wait on %s if this session's tool runner is granted it, otherwise start a watcher "+
"that rereads %s", w.Family, w.Read.Address)
}
return fmt.Sprintf("start a watcher that rereads %s", w.Read.Address)
}
// DecideStop is the Stop hook's decision, pure of IO: f and readErr are what reading the session's waits
// file gave, previous the refusal counters read beside it, and now the time. Only the first wait that still
// blocks the stop is named; the others are read again next time.
//
// A missing file (review round 2, item 1) is the normal state for a session with no waits — every stop of
// every such session, not an exceptional one — so it says nothing and simply allows, the same as a file that
// reads back with no open waits. Only a file that exists but cannot be read, or does not parse, says so.
func DecideStop(session string, f WaitsFile, readErr error, previous map[string]int, now time.Time) StopDecision {
if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) {
return StopDecision{Reason: fmt.Sprintf("the wait list could not be read (%v); allowed", readErr), Counters: map[string]int{}}
}
for _, w := range f.Waits {
blocks, reason := waitBlocksStop(session, w, now)
if !blocks {
continue
}
count := previous[w.ID] + 1
if count > MostStopRefusals {
return StopDecision{Reason: fmt.Sprintf("wait %s was refused %d times running and is let through: it was left "+
"unwatched (%s)", w.ID, MostStopRefusals, reason), Counters: map[string]int{}}
}
return StopDecision{Refuse: true, Reason: reason, Counters: map[string]int{w.ID: count}}
}
return StopDecision{Counters: map[string]int{}}
}
func refusalCountersPath(waitsDir, session string) string {
return filepath.Join(waitsDir, session+".refusals.json")
}
func readRefusalCounters(waitsDir, session string) map[string]int {
raw, err := os.ReadFile(refusalCountersPath(waitsDir, session))
if err != nil {
return map[string]int{}
}
var m map[string]int
if json.Unmarshal(raw, &m) != nil {
return map[string]int{}
}
return m
}
// writeRefusalCounters keeps the counter file beside the waits file in step with m, through fsys like every
// other write into the agent's home (novox/hq ADR 0266, issue 1749): called from the Stop hook, which
// already runs as the agent's own account (fsys is then its zero value, a plain write or remove, as
// before), and from the tool's AdoptName, which runs as the operator's account and must reach a confined
// node's agent home through the sudo helper fsys carries such a write through. Its caller decides what an
// error here means: harmless when the stop is already allowed, but when the stop was about to be refused, a
// cap that cannot be written is a cap that cannot take effect (ADR 0489 §5 "never loops", review round 1
// item 2).
//
// Empty, m is removed rather than written as "{}" (issue 1749, follow-up): a stop with nothing to record —
// every stop of every session with no waits, the ordinary case — must leave neither the file nor, the first
// time, the waits folder itself; removing a file that was never there is not an error.
func writeRefusalCounters(waitsDir, session string, m map[string]int) error {
path := refusalCountersPath(waitsDir, session)
if len(m) == 0 {
if err := fsys.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
return nil
}
raw, err := json.Marshal(m)
if err != nil {
return err
}
if err := fsys.MkdirAll(waitsDir, 0o700); err != nil {
return err
}
return fsys.WriteFile(path, raw, 0o600)
}
// notice says a fail-open or let-through reason on both the log (stderr) and the screen: Claude Code shows a
// hook's "systemMessage" on stdout, but not stderr, when the hook allows (review round 1, item a).
func notice(stdout, stderr io.Writer, reason string) {
fmt.Fprintln(stderr, reason)
writeSystemMessage(stdout, reason)
}
// runStop is the hook: exit 2 with the reason on stderr refuses the stop, exit 0 allows (1743).
func runStop(waitsDir string, stdin io.Reader, stdout, stderr io.Writer) (code int) {
defer func() {
if recover() != nil {
code = 0 // a panic always allows (1743/AC6)
}
}()
var in StopInput
if err := json.NewDecoder(stdin).Decode(&in); err != nil {
return 0
}
sessionID := strings.TrimSpace(in.SessionID)
if sessionID == "" {
return 0
}
// Review round 1, item 1: a session id is never used to build a path until it is checked against
// waitSessionID — no "/" in it reaches the filesystem, so it can never name a file outside waitsDir.
if !waitSessionID.MatchString(sessionID) {
notice(stdout, stderr, fmt.Sprintf("the session id %q could not be used to find its waits (letters, digits, "+
"'.', '_' and '-' only); this stop is allowed", sessionID))
return 0
}
f, err := readWaitsFile(waitsFileIn(waitsDir, sessionID))
counters := readRefusalCounters(waitsDir, sessionID)
d := DecideStop(sessionID, f, err, counters, time.Now())
if werr := writeRefusalCounters(waitsDir, sessionID, d.Counters); werr != nil && d.Refuse {
// Item 2: writing the counter failed on the very call that would have incremented it toward the cap;
// refusing anyway could hold the turn forever, so this call is let through instead.
notice(stdout, stderr, fmt.Sprintf("this wait's refusal counter could not be written (%v); allowed so it "+
"cannot hold the turn forever", werr))
return 0
}
if d.Reason != "" {
fmt.Fprintln(stderr, d.Reason)
if !d.Refuse {
writeSystemMessage(stdout, d.Reason) // item a: a fail-open or let-through notice also reaches the screen
}
}
if d.Refuse {
return 2
}
return 0
}
@@ -0,0 +1,362 @@
package main
// The plugin's Stop hook (novox/hq ADR 0489 §3-§6, issue 1743).
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
var stopT0 = time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)
func openWait(id string, bound int) Wait {
return Wait{ID: id, What: "the operator's Approve", Bound: bound, At: stopT0}
}
// 1743/AC1: a wait within its bound and unwatched refuses the stop, naming the wait, its read address and
// what the session does about it (mesh_wait on its family, conditionally — review round 1 item 3 — or a
// watcher that rereads the address).
func TestStopRefusesAnUnwatchedWaitWithinItsBound(t *testing.T) {
w := openWait("w-1", 10)
w.Read = WaitRead{Address: "mesh-delivery.recheck"}
w.Family = "mesh-delivery.decided"
d := DecideStop("sess-1", WaitsFile{Waits: []Wait{w}}, nil, map[string]int{}, stopT0.Add(time.Minute))
if !d.Refuse || !strings.Contains(d.Reason, "w-1") || !strings.Contains(d.Reason, "mesh-delivery.recheck") ||
!strings.Contains(d.Reason, "mesh_wait") || !strings.Contains(d.Reason, "watcher") {
t.Fatalf("%+v", d)
}
}
// 1743/AC2: a watched wait within its bound, a done wait (absent from the file), or no file entry: allowed.
func TestStopAllowsAWatchedOrDoneOrAbsentWait(t *testing.T) {
watched := openWait("w-2", 10)
watched.Watcher = "mesh_wait on mesh-delivery.decided"
if d := DecideStop("sess-2", WaitsFile{Waits: []Wait{watched}}, nil, map[string]int{}, stopT0.Add(time.Minute)); d.Refuse {
t.Fatalf("a watched wait within its bound was refused: %+v", d)
}
if d := DecideStop("sess-2", WaitsFile{Waits: []Wait{}}, nil, map[string]int{}, stopT0); d.Refuse {
t.Fatalf("no open waits was refused: %+v", d)
}
}
// 1743/AC3: a wait past its bound, watched or not, is refused until said, then allowed.
func TestStopRefusesPastBoundUntilSaidWatchedOrNot(t *testing.T) {
unwatched := openWait("w-3", 5)
if d := DecideStop("sess-3", WaitsFile{Waits: []Wait{unwatched}}, nil, map[string]int{}, stopT0.Add(10*time.Minute)); !d.Refuse ||
!strings.Contains(d.Reason, "w-3") || !strings.Contains(d.Reason, "said") {
t.Fatalf("unwatched past bound: %+v", d)
}
watched := unwatched
watched.Watcher = "mesh_wait on mesh-delivery.decided"
if d := DecideStop("sess-3", WaitsFile{Waits: []Wait{watched}}, nil, map[string]int{}, stopT0.Add(10*time.Minute)); !d.Refuse {
t.Fatalf("watched past bound was allowed: %+v", d)
}
said := watched
said.Said = true
if d := DecideStop("sess-3", WaitsFile{Waits: []Wait{said}}, nil, map[string]int{}, stopT0.Add(10*time.Minute)); d.Refuse {
t.Fatalf("said past bound was still refused: %+v", d)
}
}
// 1743/AC4: an unreadable or malformed waits file is allowed, saying so on stderr and as a systemMessage on
// stdout; the decision does not touch the network and answers at once.
func TestStopAllowsAndSaysSoWhenTheWaitsFileIsUnreadableOrMalformed(t *testing.T) {
d := DecideStop("sess-4", WaitsFile{}, errors.New("no such file"), map[string]int{}, stopT0)
if d.Refuse || !strings.Contains(d.Reason, "could not be read") {
t.Fatalf("%+v", d)
}
waitsDir := t.TempDir()
if err := os.WriteFile(filepath.Join(waitsDir, "sess-4b.json"), []byte("{not json"), 0o600); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(`{"session_id":"sess-4b","stop_hook_active":false}`)
start := time.Now()
code := runStop(waitsDir, stdin, &stdout, &stderr)
if code != 0 || !strings.Contains(stderr.String(), "could not be read") {
t.Fatalf("code %d, stderr %s", code, stderr.String())
}
// Item (a): a fail-open notice also reaches the screen, as the one JSON object Claude Code shows.
if !strings.Contains(stdout.String(), "systemMessage") || !strings.Contains(stdout.String(), "could not be read") {
t.Fatalf("the fail-open notice did not reach stdout: %s", stdout.String())
}
if took := time.Since(start); took > 5*time.Second {
t.Fatalf("took %s; the Stop hook has no network and is bounded under 5s", took)
}
}
// Review round 2, item 1 (1743/AC2, ADR 0490 §2): a missing waits file is the normal state for a session
// with no waits — every stop of every such session — so it must print nothing, on either stream, and
// simply allow; a notice there would fire constantly for nothing wrong.
func TestStopPrintsNothingWhenTheWaitsFileIsSimplyMissing(t *testing.T) {
d := DecideStop("sess-4c", WaitsFile{}, fs.ErrNotExist, map[string]int{}, stopT0)
if d.Refuse || d.Reason != "" {
t.Fatalf("a missing file (fs.ErrNotExist) said something: %+v", d)
}
waitsDir := t.TempDir() // no file at all for sess-4d
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(`{"session_id":"sess-4d","stop_hook_active":false}`)
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 0 {
t.Fatalf("code %d", code)
}
if stdout.Len() != 0 || stderr.Len() != 0 {
t.Fatalf("a missing waits file printed something: stdout %q, stderr %q", stdout.String(), stderr.String())
}
}
// 1743/AC5: the same stop refused three times running for the same wait is allowed the fourth time, saying
// it was left unwatched; the counter resets when the wait changes.
func TestStopAllowsTheFourthRefusalInARow(t *testing.T) {
w := openWait("w-5", 10)
counters := map[string]int{}
for i := 1; i <= MostStopRefusals; i++ {
d := DecideStop("sess-5", WaitsFile{Waits: []Wait{w}}, nil, counters, stopT0.Add(time.Minute))
if !d.Refuse {
t.Fatalf("refusal %d was allowed: %+v", i, d)
}
counters = d.Counters
}
d := DecideStop("sess-5", WaitsFile{Waits: []Wait{w}}, nil, counters, stopT0.Add(time.Minute))
if d.Refuse || !strings.Contains(d.Reason, "unwatched") {
t.Fatalf("the fourth refusal was not let through: %+v", d)
}
if len(d.Counters) != 0 {
t.Fatalf("the counter was not reset once let through: %+v", d.Counters)
}
// A different wait starts its own count from zero.
w2 := openWait("w-5b", 10)
d2 := DecideStop("sess-5", WaitsFile{Waits: []Wait{w2}}, nil, counters, stopT0.Add(time.Minute))
if !d2.Refuse || d2.Counters["w-5b"] != 1 {
t.Fatalf("a different wait did not start its own count: %+v", d2)
}
}
// 1743/AC6: a panic answers allow.
func TestStopAPanicAllows(t *testing.T) {
waitsDir := t.TempDir()
var stdout, stderr bytes.Buffer
if code := runStop(waitsDir, panicReader{}, &stdout, &stderr); code != 0 {
t.Fatalf("a panic was not recovered: code %d", code)
}
}
// runStop through its counter file: three refusals in a row for the one open wait, then let through.
func TestRunStopKeepsItsCounterBesideTheWaitsFile(t *testing.T) {
waitsDir := t.TempDir()
f := WaitsFile{Session: "sess-6", At: stopT0, Waits: []Wait{openWait("w-6", 10)}}
writeWaitsFixture(t, filepath.Join(waitsDir, "sess-6.json"), f)
stdin := func() *strings.Reader { return strings.NewReader(`{"session_id":"sess-6","stop_hook_active":false}`) }
for i := 1; i <= MostStopRefusals; i++ {
var stdout, stderr bytes.Buffer
if code := runStop(waitsDir, stdin(), &stdout, &stderr); code != 2 {
t.Fatalf("refusal %d: code %d, stderr %s", i, code, stderr.String())
}
}
var stdout, stderr bytes.Buffer
if code := runStop(waitsDir, stdin(), &stdout, &stderr); code != 0 || !strings.Contains(stderr.String(), "unwatched") {
t.Fatalf("the fourth call: code %d, stderr %s", code, stderr.String())
}
// Item (a): the let-through notice reaches the screen too, not only the log.
if !strings.Contains(stdout.String(), "systemMessage") || !strings.Contains(stdout.String(), "unwatched") {
t.Fatalf("the let-through notice did not reach stdout: %s", stdout.String())
}
// Once let through, the counters are empty again: the file is removed, not left holding "{}" (issue
// 1749, follow-up).
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-6")); !os.IsNotExist(err) {
t.Fatalf("the counter file was not removed once its counters emptied: %v", err)
}
}
// Issue 1749, follow-up: a stop for a session with no waits at all leaves nothing behind — not the counter
// file, not even the waits folder a node that never added a wait would otherwise gain on every stop of
// every session.
func TestStopWithNoWaitsLeavesNoFolderAndNoFile(t *testing.T) {
waitsDir := filepath.Join(t.TempDir(), "waits") // not created: this is what the hook is handed cold
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(`{"session_id":"sess-none","stop_hook_active":false}`)
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 0 {
t.Fatalf("code %d, stderr %s", code, stderr.String())
}
if _, err := os.Stat(waitsDir); !os.IsNotExist(err) {
t.Fatalf("a stop with no waits created the waits folder: %v", err)
}
}
// Issue 1749, follow-up: a refusal writes the counter file (there is something to record).
func TestStopWithARefusalWritesTheCounterFile(t *testing.T) {
waitsDir := t.TempDir()
f := WaitsFile{Session: "sess-7", At: stopT0, Waits: []Wait{openWait("w-7", 10)}}
writeWaitsFixture(t, filepath.Join(waitsDir, "sess-7.json"), f)
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(`{"session_id":"sess-7","stop_hook_active":false}`)
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 2 {
t.Fatalf("code %d, stderr %s", code, stderr.String())
}
raw, err := os.ReadFile(refusalCountersPath(waitsDir, "sess-7"))
if err != nil {
t.Fatal(err)
}
var counters map[string]int
if json.Unmarshal(raw, &counters) != nil || counters["w-7"] != 1 {
t.Fatalf("the counter file: %s", raw)
}
}
// Issue 1749, follow-up: once the wait is done (no longer in the waits file), the next stop removes the
// counter file rather than leaving a stale "{}" behind.
func TestStopAfterTheWaitIsDoneRemovesTheCounterFile(t *testing.T) {
waitsDir := t.TempDir()
path := filepath.Join(waitsDir, "sess-8.json")
writeWaitsFixture(t, path, WaitsFile{Session: "sess-8", At: stopT0, Waits: []Wait{openWait("w-8", 10)}})
stdin := strings.NewReader(`{"session_id":"sess-8","stop_hook_active":false}`)
var stdout, stderr bytes.Buffer
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 2 {
t.Fatalf("refusal: code %d, stderr %s", code, stderr.String())
}
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-8")); err != nil {
t.Fatalf("the counter file was not written: %v", err)
}
// The wait is done: it is no longer in the waits file (DoneWait removes it, waits.go).
writeWaitsFixture(t, path, WaitsFile{Session: "sess-8", At: stopT0, Waits: []Wait{}})
var stdout2, stderr2 bytes.Buffer
stdin2 := strings.NewReader(`{"session_id":"sess-8","stop_hook_active":false}`)
if code := runStop(waitsDir, stdin2, &stdout2, &stderr2); code != 0 {
t.Fatalf("after done: code %d, stderr %s", code, stderr2.String())
}
if _, err := os.Stat(refusalCountersPath(waitsDir, "sess-8")); !os.IsNotExist(err) {
t.Fatalf("the counter file was not removed after the wait was done: %v", err)
}
}
// Review round 1, item 1 (1743/1744, ADR 0489 §5): a session id that is not a plain one (path separators,
// ".."), is never used to build a file path — the counter file stays inside the waits directory — and the
// Stop hook allows, saying why.
func TestStopRejectsAnUnsafeSessionID(t *testing.T) {
for _, id := range []string{"../../x", "a/b"} {
root := t.TempDir()
waitsDir := filepath.Join(root, "agent", "waits")
if err := os.MkdirAll(waitsDir, 0o700); err != nil {
t.Fatal(err)
}
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(fmt.Sprintf(`{"session_id":%q,"stop_hook_active":false}`, id))
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 0 {
t.Fatalf("id %q: code %d, stderr %s", id, code, stderr.String())
}
if !strings.Contains(stderr.String(), "could not be used") {
t.Fatalf("id %q: did not say why it allowed: %s", id, stderr.String())
}
if !strings.Contains(stdout.String(), "systemMessage") {
t.Fatalf("id %q: the notice did not reach stdout: %s", id, stdout.String())
}
escaped := filepath.Join(waitsDir, id+".refusals.json")
if _, err := os.Stat(escaped); !os.IsNotExist(err) {
t.Fatalf("id %q: a counter file was written outside the waits dir at %s", id, escaped)
}
}
// A blank id still just allows, quietly: it never reaches the point of building a path at all.
waitsDir := t.TempDir()
var stdout, stderr bytes.Buffer
if code := runStop(waitsDir, strings.NewReader(`{"session_id":""}`), &stdout, &stderr); code != 0 {
t.Fatalf("blank id: code %d", code)
}
}
// Review round 1, item 2 (ADR 0489 §5 "never loops", 1743/AC5): when the decision is to refuse and the
// refusal counter cannot be written, the hook allows rather than risk refusing forever with a cap that
// never takes effect.
func TestStopAllowsWhenTheRefusalCounterCannotBeWritten(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("running as root: a directory's write bit cannot be taken away")
}
waitsDir := t.TempDir()
writeWaitsFixture(t, filepath.Join(waitsDir, "sess-7.json"),
WaitsFile{Session: "sess-7", At: stopT0, Waits: []Wait{openWait("w-7", 10)}})
if err := os.Chmod(waitsDir, 0o500); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chmod(waitsDir, 0o700) })
var stdout, stderr bytes.Buffer
stdin := strings.NewReader(`{"session_id":"sess-7","stop_hook_active":false}`)
if code := runStop(waitsDir, stdin, &stdout, &stderr); code != 0 {
t.Fatalf("code %d, stderr %s", code, stderr.String())
}
if !strings.Contains(stderr.String(), "counter") {
t.Fatalf("did not say the counter could not be written: %s", stderr.String())
}
if !strings.Contains(stdout.String(), "systemMessage") {
t.Fatalf("the notice did not reach stdout: %s", stdout.String())
}
}
// Review round 1, item 3 (1743/AC1, §3 with ADR 0367 §2): a family is offered only conditionally (the tool
// runner may not be granted it); without one, the advice is only to start a watcher.
func TestWaitBlocksStopAdvisesMeshWaitOnlyConditionallyOtherwiseAWatcher(t *testing.T) {
withFamily := openWait("w-f", 10)
withFamily.Read = WaitRead{Address: "mesh-delivery.recheck"}
withFamily.Family = "mesh-delivery.decided"
_, reason := waitBlocksStop("sess-f", withFamily, stopT0.Add(time.Minute))
if !strings.Contains(reason, "mesh_wait on mesh-delivery.decided if this session's tool runner is granted it") ||
!strings.Contains(reason, "otherwise start a watcher that rereads mesh-delivery.recheck") {
t.Fatalf("%s", reason)
}
noFamily := openWait("w-nf", 10)
noFamily.Read = WaitRead{Address: "mesh-delivery.recheck"}
_, reason2 := waitBlocksStop("sess-nf", noFamily, stopT0.Add(time.Minute))
if strings.Contains(reason2, "mesh_wait") || !strings.Contains(reason2, "start a watcher that rereads mesh-delivery.recheck") {
t.Fatalf("%s", reason2)
}
}
// 1743/AC7's rendering half (hooks.json holds the guard's PreToolUse entry and the Stop entry) is checked
// together with 1744/AC4 in terminal_only_test.go's
// TestThePluginsOnlyHooksAreTheGuardTheSessionStartHookAndTheStopHook. claude_code_hook_register still
// refuses a Stop hook (or any other) at every scope.
func TestHookRegisterStillRefusesEveryHook(t *testing.T) {
p, _ := node(t, "laptop")
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isError := s.call(t, "claude_code_hook_register", map[string]any{"name": "x", "event": "Stop", "command": "true"})
if !isError || !strings.Contains(text, "issue 340") {
t.Fatalf("a Stop hook was not refused: %v %s", isError, text)
}
}
// Issue 1749: adopt's refusal-counter reset writes into the agent's home (beside the waits file) and must
// go through fsys like every other such write (novox/hq ADR 0266) — on a confined node the module runs as
// the operator's account, which cannot write there directly; only the helper, as the agent's own account,
// can. The confined-node fake (agentFiles, agent_account_test.go) proves it: a write that reaches the real
// filesystem only through the helper's request/response protocol bumps its count.
func TestWriteRefusalCountersGoesThroughTheHelper(t *testing.T) {
f, home, counted := agentFiles(t)
was := fsys
fsys = f
t.Cleanup(func() { fsys = was })
waitsDir := filepath.Join(home, ".claude", "nox-mesh", "waits")
if err := writeRefusalCounters(waitsDir, "sess-1", map[string]int{"w-1": 2}); err != nil {
t.Fatal(err)
}
if counted.asked.Load() == 0 {
t.Fatal("the counter file was written directly, never through the helper")
}
raw, err := os.ReadFile(filepath.Join(waitsDir, "sess-1.refusals.json"))
if err != nil {
t.Fatal(err)
}
var m map[string]int
if json.Unmarshal(raw, &m) != nil || m["w-1"] != 2 {
t.Fatalf("the written counters: %s", raw)
}
}
@@ -261,12 +261,12 @@ func TestTheOperatorsManagedSettingsAreWhatSessionsObey(t *testing.T) {
}
}
// The plugin's hooks are the mesh's own guard on the shell (novox/hq ADR 0245) and nothing else, whatever the
// configuration given holds (novox/hq issue 340).
func TestThePluginsOnlyHookIsTheGuard(t *testing.T) {
// The plugin's only hooks are the guard, the session-start hook and the Stop hook (novox/hq ADR 0245, ADR
// 0489) — never a hook a registration named, whatever the configuration given holds (novox/hq issue 340).
func TestThePluginsOnlyHooksAreTheGuardTheSessionStartHookAndTheStopHook(t *testing.T) {
c := Config{Mesh: []Item{{Kind: KindHook, Name: "planted", Scope: ScopeMesh, Event: "PreToolUse", Matcher: "Bash",
Command: "${HOOK_DIR}/x.sh", Files: map[string]string{"x.sh": "#!/bin/sh\n"}}}}
plugin := MarketplaceWith(c, &GuardFiles{Exe: "/bin/true"})
plugin := MarketplaceWith(c, &GuardFiles{Exe: "/bin/true", WaitsDir: "/home/agent/.claude/nox-mesh/waits"})
root := "plugins/" + Plugin + "/"
var hooks struct {
Hooks map[string][]struct {
@@ -279,9 +279,16 @@ func TestThePluginsOnlyHookIsTheGuard(t *testing.T) {
if err := json.Unmarshal([]byte(plugin[root+"hooks/hooks.json"].Content), &hooks); err != nil {
t.Fatal(err)
}
if len(hooks.Hooks) != 1 || len(hooks.Hooks["PreToolUse"]) != 1 || hooks.Hooks["PreToolUse"][0].Matcher != GuardMatcher ||
if len(hooks.Hooks) != 3 || len(hooks.Hooks["PreToolUse"]) != 1 || hooks.Hooks["PreToolUse"][0].Matcher != GuardMatcher ||
hooks.Hooks["PreToolUse"][0].Hooks[0].Command != GuardCommand() {
t.Fatalf("the plugin's hooks: %s", plugin[root+"hooks/hooks.json"].Content)
t.Fatalf("the guard's hook: %s", plugin[root+"hooks/hooks.json"].Content)
}
if len(hooks.Hooks["SessionStart"]) != 1 || hooks.Hooks["SessionStart"][0].Matcher != "startup|resume|clear|compact" ||
hooks.Hooks["SessionStart"][0].Hooks[0].Command != SessionStartCommand("/home/agent/.claude/nox-mesh/waits") {
t.Fatalf("the session-start hook: %s", plugin[root+"hooks/hooks.json"].Content)
}
if len(hooks.Hooks["Stop"]) != 1 || hooks.Hooks["Stop"][0].Hooks[0].Command != StopCommand("/home/agent/.claude/nox-mesh/waits") {
t.Fatalf("the Stop hook: %s", plugin[root+"hooks/hooks.json"].Content)
}
for path := range plugin {
if strings.HasPrefix(path, root+"hooks/planted") {
@@ -0,0 +1,776 @@
package main
// A session's wait list (novox/hq ADR 0489, as changed by ADR 0490): what a session still expects an answer
// to — an Approve, a delivery's end, anything it started and must act on when it ends
// (follow-what-you-start) — so the plugin's Stop hook can refuse to end its turn while one is open and
// nothing watches it.
//
// Kept in the module's `waits` state under the session's name and this node (ADR 0490 §1) — `by`, as
// claude_code_ask takes it, never the session id, which Claude Code hands a session a new one of at a
// /clear. Every call also gives `session`, the caller's current id; the module keeps the binding of a name
// to the session that last called for it, with that call's time, so the same name's waits are found again
// across a /clear once the new session has adopted it.
//
// The waits file a hook reads is kept per session id, not per name, and holds the waits of whichever name is
// currently bound to that id (empty for an id nothing is bound to): written whole, atomically, on every
// change, and rewritten for both ids when a binding moves (AdoptName). A second file, the node-wide index,
// lists every name on this node that still holds an open wait, so the SessionStart hook can tell a session
// with a brand new id what it might want to adopt, without calling back into this module.
//
// Every file lives under the agent's home (the manifest's `agent-home` resource), which this process —
// launched as the operator's account (novox/hq ADR 0193) — cannot itself write when the node names an agent
// account of its own (novox/hq ADR 0266): every write below goes through `fsys` (homefs.go), which already
// carries such a write through the sudo helper the guard's binary also serves, exactly as grant.go's
// WriteCredentials does for the agent's own credentials file. Where the node has no agent account of its
// own, fsys is the plain filesystem and the write reaches the operator's own home directly.
import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// WaitsState is the module's state the wait lists are kept in.
const WaitsState = "waits"
// WaitsDir is where the mirrored files live under the agent's home.
const WaitsDir = "nox-mesh/waits"
// WaitPrefix starts every wait's id.
const WaitPrefix = "w-"
// MostWaits is how many open waits one name may hold at once (novox/hq ADR 0489, 1629/AC5).
const MostWaits = 50
// The bounds of a wait's bound, in minutes (1629/AC5).
const (
ShortestWaitBound = 1
LongestWaitBound = 1440
)
// mostWaitAttempts bounds the compare-and-set retry loop over the `waits` state.
const mostWaitAttempts = 20
var waitSessionID = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$`)
// requireSessionID trims session and refuses it unless it is a plain id matching waitSessionID — every op
// holds to this (review round 1, item b), so a session id is never used to build a path from anything but a
// safe name (no "/", so it cannot name a file outside the waits directory).
func requireSessionID(session string) (string, error) {
session = strings.TrimSpace(session)
if !waitSessionID.MatchString(session) {
return "", errors.New("session is required: the id the SessionStart hook named")
}
return session, nil
}
// WaitRead is the verb a session reads to see where one wait stands.
type WaitRead struct {
Address string `json:"address"`
Arguments map[string]any `json:"arguments,omitempty"`
}
// Wait is one answer a session wrote down that it expects (novox/hq ADR 0489 §1-§2). By is the name it was
// added under.
type Wait struct {
ID string `json:"id"`
By string `json:"by"`
What string `json:"what"`
Read WaitRead `json:"read"`
Family string `json:"family,omitempty"`
Bound int `json:"bound"` // minutes
At time.Time `json:"at"`
Watcher string `json:"watcher,omitempty"`
Said bool `json:"said,omitempty"`
}
// PastBound says whether now is past this wait's bound.
func (w Wait) PastBound(now time.Time) bool {
return now.Sub(w.At) > time.Duration(w.Bound)*time.Minute
}
// NamedWaits is what the `waits` state keeps for one name on this node (novox/hq ADR 0490 §1): its open
// waits, and the binding to whichever session currently answers to that name — empty until the first call
// names it, by add or by adopt.
type NamedWaits struct {
Name string `json:"name"`
Node string `json:"node"`
CurrentSession string `json:"currentSession,omitempty"`
LastCall time.Time `json:"lastCall,omitempty"`
Waits []Wait `json:"waits"`
}
// WaitsFile is what one session id's mirrored file holds, whole, on every change (1629/AC3): the open waits
// of whichever name is currently bound to this session, and the time they were written.
type WaitsFile struct {
Session string `json:"session"`
At time.Time `json:"at"`
Waits []Wait `json:"waits"`
}
// NodeIndexEntry is one name's open waits, as the node-wide index lists it for the SessionStart hook
// (novox/hq ADR 0490 §2).
type NodeIndexEntry struct {
Name string `json:"name"`
Open int `json:"open"`
CurrentSession string `json:"currentSession,omitempty"`
LastCall time.Time `json:"lastCall"`
}
// NodeIndex is the node-wide index file's whole content (novox/hq ADR 0490 §2): every name on this node that
// still holds an open wait, written by the tool on every change a name's waits or binding takes, so the
// SessionStart hook can say "if this is you, adopt" without calling back into this module.
type NodeIndex struct {
Node string `json:"node"`
At time.Time `json:"at"`
Names []NodeIndexEntry `json:"names"`
}
// WaitStore is the `waits` state: a value with its revision, created only when absent, updated only at the
// revision read; Keys lists every name kept, for the node-wide index.
type WaitStore interface {
Get(key string) (json.RawMessage, uint64, bool, error)
Create(key string, value any) error
Update(key string, value any, revision uint64) error
Keys() ([]string, error)
}
// keptWaits is the `waits` state on the bus.
type keptWaits struct{ s stdio.KeptState }
func (k keptWaits) Get(key string) (json.RawMessage, uint64, bool, error) {
e, err := k.s.Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (k keptWaits) Create(key string, value any) error {
_, err := k.s.Create(key, value)
return err
}
func (k keptWaits) Update(key string, value any, revision uint64) error {
_, err := k.s.Update(key, value, revision)
return err
}
func (k keptWaits) Keys() ([]string, error) { return k.s.Keys() }
// Waiting is what claude_code_wait reaches: the `waits` state and the clock. Nil outside a launch.
type Waiting struct {
Store WaitStore
Now func() time.Time
}
var waiting *Waiting
// waitsKey is the state's key for one name's waits: its name and this node (novox/hq ADR 0490 §1), so two
// nodes never collide over the same name.
func waitsKey(p Paths, name string) string { return p.Node + "/" + name }
// WaitsDirPath is where this node's sessions' waits are mirrored, under the agent's home: the directory the
// plugin's SessionStart and Stop hooks (1744, 1743) are given on their command line.
func WaitsDirPath(p Paths) string {
return filepath.Join(p.Home, ".claude", filepath.FromSlash(WaitsDir))
}
// waitsFilePath is where one session id's mirrored waits are.
func waitsFilePath(p Paths, session string) string {
return filepath.Join(WaitsDirPath(p), session+".json")
}
// nodeIndexPath is the node-wide index's file, beside the per-session ones; its name can never collide with
// a session id, since waitSessionID never allows a leading "_".
func nodeIndexPath(p Paths) string { return filepath.Join(WaitsDirPath(p), "_index.json") }
func newWaitID() string {
var b [8]byte
_, _ = rand.Read(b[:])
return WaitPrefix + hex.EncodeToString(b[:])
}
// addressShaped says whether s could be an address `mesh_call` takes: not blank, no whitespace, and a dot
// between two names (`<seat>.<verb>`, `<node>/<seat>.<verb>`, `<node>/<module>.<tool>`).
func addressShaped(s string) bool {
s = strings.TrimSpace(s)
if s == "" || strings.ContainsAny(s, " \t\n\r") {
return false
}
dot := strings.LastIndex(s, ".")
return dot > 0 && dot < len(s)-1
}
func readNamedWaits(store WaitStore, key string) (NamedWaits, uint64, bool, error) {
raw, rev, found, err := store.Get(key)
if err != nil || !found {
return NamedWaits{}, 0, found, err
}
var nw NamedWaits
_ = json.Unmarshal(raw, &nw)
return nw, rev, true, nil
}
// updateNamedWaits reads one name's waits, lets mutate change them, and writes the result back at the
// revision read — created when there was none yet — retried when another write won the race.
func updateNamedWaits(store WaitStore, key string, mutate func(NamedWaits) (NamedWaits, error)) (NamedWaits, error) {
for i := 0; i < mostWaitAttempts; i++ {
nw, rev, found, err := readNamedWaits(store, key)
if err != nil {
return NamedWaits{}, fmt.Errorf("the wait list could not be read: %w", err)
}
next, err := mutate(nw)
if err != nil {
return NamedWaits{}, err
}
if !found {
err = store.Create(key, next)
} else {
err = store.Update(key, next, rev)
}
if err == nil {
return next, nil
}
if errors.Is(err, stdio.ErrChanged) {
continue
}
return NamedWaits{}, fmt.Errorf("the wait list could not be written: %w", err)
}
return NamedWaits{}, errors.New("the wait list kept changing elsewhere; try again")
}
// tempSuffix is a short random name for a temp file, so two writes racing for the same path never share one
// (review round 2, item a): each gets its own, and only the rename is a single atomic step.
func tempSuffix() string {
var b [8]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
// writeSessionFile rewrites one session id's mirrored file whole, atomically (its own temp file, then
// rename): it holds exactly the open waits given and the time of this write (1629/AC3). waits is always the
// union of every name currently bound to sessionID (sessionWaitsUnion), never only the name an op acted on:
// a second name bound to the same id must stay enforced (ADR 0490 §1, review round 2 item 2).
func writeSessionFile(p Paths, sessionID string, waits []Wait, now time.Time) error {
path := waitsFilePath(p, sessionID)
if err := fsys.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
if waits == nil {
waits = []Wait{}
}
raw, err := indented(WaitsFile{Session: sessionID, At: now.UTC(), Waits: waits})
if err != nil {
return err
}
tmp := path + ".mesh-tmp-" + tempSuffix()
if err := fsys.WriteFile(tmp, raw, 0o600); err != nil {
return err
}
return fsys.Rename(tmp, path)
}
// sessionWaitsUnion is every open wait of every name on this node currently bound to session (ADR 0490 §1):
// what that session id's mirrored file must hold, whole — never only the one name an op just acted on, or a
// second name bound to the same id would stop being enforced the moment any op touched the first.
func sessionWaitsUnion(store WaitStore, node, session string) ([]Wait, error) {
keys, err := store.Keys()
if err != nil {
return nil, err
}
prefix := node + "/"
var out []Wait
names := make([]string, 0, len(keys))
for _, k := range keys {
if strings.HasPrefix(k, prefix) {
names = append(names, k)
}
}
sort.Strings(names) // deterministic order: tests, and a session reading its own file, see the same thing
for _, k := range names {
nw, _, found, err := readNamedWaits(store, k)
if err != nil || !found || nw.CurrentSession != session {
continue
}
out = append(out, nw.Waits...)
}
return out, nil
}
// writeSessionWaits rewrites session's mirrored file with sessionWaitsUnion: every call that changes a
// name's waits or its binding goes through this, never writeSessionFile directly with one name's waits
// alone.
func writeSessionWaits(p Paths, store WaitStore, session string, now time.Time) error {
union, err := sessionWaitsUnion(store, p.Node, session)
if err != nil {
return fmt.Errorf("the session's other names could not be read: %w", err)
}
return writeSessionFile(p, session, union, now)
}
// writeNodeIndex rewrites the node-wide index whole, atomically, from every name's current state on this
// node (novox/hq ADR 0490 §2): a full scan of the `waits` state, bounded by how many names are ever used on
// one node at once. Best-effort for its caller: a name's own file is already correct by the time this runs.
func writeNodeIndex(p Paths, store WaitStore, now time.Time) error {
keys, err := store.Keys()
if err != nil {
return fmt.Errorf("the node-wide wait index could not be read: %w", err)
}
prefix := p.Node + "/"
idx := NodeIndex{Node: p.Node, At: now.UTC(), Names: []NodeIndexEntry{}}
for _, k := range keys {
name, ok := strings.CutPrefix(k, prefix)
if !ok {
continue
}
nw, _, found, err := readNamedWaits(store, k)
if err != nil || !found || len(nw.Waits) == 0 {
continue
}
idx.Names = append(idx.Names, NodeIndexEntry{Name: name, Open: len(nw.Waits),
CurrentSession: nw.CurrentSession, LastCall: nw.LastCall.UTC()})
}
sort.Slice(idx.Names, func(i, j int) bool { return idx.Names[i].Name < idx.Names[j].Name })
path := nodeIndexPath(p)
if err := fsys.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
raw, err := indented(idx)
if err != nil {
return err
}
tmp := path + ".mesh-tmp-" + tempSuffix()
if err := fsys.WriteFile(tmp, raw, 0o600); err != nil {
return err
}
return fsys.Rename(tmp, path)
}
// callerName reads `by` as claude_code_ask does: who@node, where node is this one — the name a wait list is
// kept under (novox/hq ADR 0490 §1), never the session id.
func callerName(p Paths, by string) (string, error) {
by = strings.TrimSpace(by)
if who, node, found := strings.Cut(by, "@"); found {
if node != p.Node {
return "", fmt.Errorf("by names the node %q, and this is %s: call on the node the session runs on", node, p.Node)
}
return who, nil
}
if by == "" {
return "", errors.New("by is required: your session's name, as who@node or who")
}
return by, nil
}
// bindCaller checks the binding for name against session, refusing when another, more recently bound
// session holds it (novox/hq ADR 0490 §3) — naming that session and when it last called, and how to take the
// name back with adopt — and otherwise (re)binds it to session at now, so a name nothing has bound yet is
// claimed by the first session that calls for it, add included.
func bindCaller(nw *NamedWaits, name, node, by, session string, now time.Time) error {
if nw.CurrentSession != "" && nw.CurrentSession != session {
return fmt.Errorf("the name %q now answers to session %s (it last called at %s); adopt it again if this "+
"is you: claude_code_wait {\"op\":\"adopt\",\"by\":%q,\"session\":%q}", name, nw.CurrentSession,
nw.LastCall.UTC().Format(time.RFC3339), by, session)
}
nw.Name, nw.Node, nw.CurrentSession, nw.LastCall = name, node, session, now.UTC()
return nil
}
// findWait is the index of id in waits, or -1.
func findWait(waits []Wait, id string) int {
for i, w := range waits {
if w.ID == id {
return i
}
}
return -1
}
// WaitInput is what claude_code_wait's add takes.
type WaitInput struct {
Session string
By string
What string
Read WaitRead
Family string
Bound int
}
// AddWait keeps one more wait for a name, refusing outside ADR 0489's bounds (1629/AC5) or when another
// session has since taken the name over (ADR 0490 §3), and rewrites the caller's mirrored file (1629/AC3).
func AddWait(p Paths, in WaitInput, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(in.Session)
if err != nil {
return nil, err
}
name, err := callerName(p, in.By)
if err != nil {
return nil, err
}
what := strings.TrimSpace(in.What)
if what == "" {
return nil, errors.New("what is required: say what you are waiting for")
}
if in.Bound < ShortestWaitBound || in.Bound > LongestWaitBound {
return nil, fmt.Errorf("bound_minutes is %d; it is %d to %d", in.Bound, ShortestWaitBound, LongestWaitBound)
}
address := strings.TrimSpace(in.Read.Address)
if !addressShaped(address) {
return nil, fmt.Errorf("read.address %q is not an address: the verb that reads where this wait stands, as mesh_call takes it", in.Read.Address)
}
w := Wait{ID: newWaitID(), By: name, What: what, Read: WaitRead{Address: address, Arguments: in.Read.Arguments},
Family: strings.TrimSpace(in.Family), Bound: in.Bound, At: now.UTC()}
key := waitsKey(p, name)
nw, err := updateNamedWaits(store, key, func(nw NamedWaits) (NamedWaits, error) {
if err := bindCaller(&nw, name, p.Node, in.By, session, now); err != nil {
return nw, err
}
if len(nw.Waits) >= MostWaits {
return nw, fmt.Errorf("%d waits already wait for %s; at most %d open waits per name", len(nw.Waits), name, MostWaits)
}
nw.Waits = append(append([]Wait(nil), nw.Waits...), w)
return nw, nil
})
if err != nil {
return nil, err
}
if err := writeSessionWaits(p, store, session, now); err != nil {
return nil, fmt.Errorf("the wait was kept, but its file could not be written: %w", err)
}
if err := writeNodeIndex(p, store, now); err != nil {
say("writing the node-wide wait index after %s added for %s: %v", w.ID, name, err)
}
say("wait %s added for %s (session %s): %s", w.ID, name, session, what)
return map[string]any{"id": w.ID, "name": name, "session": session, "open": len(nw.Waits)}, nil
}
// WatchedWait marks a wait as followed: by mesh_wait on its family, or a watcher started for it (1629/AC2).
func WatchedWait(p Paths, session, by, id, watcher string, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(session)
if err != nil {
return nil, err
}
name, err := callerName(p, by)
if err != nil {
return nil, err
}
id, watcher = strings.TrimSpace(id), strings.TrimSpace(watcher)
if id == "" {
return nil, errors.New("id is required: the one add answered")
}
if watcher == "" {
return nil, errors.New("watcher is required: what follows this wait (mesh_wait on its family, or a watcher agent)")
}
key := waitsKey(p, name)
_, err = updateNamedWaits(store, key, func(nw NamedWaits) (NamedWaits, error) {
if err := bindCaller(&nw, name, p.Node, by, session, now); err != nil {
return nw, err
}
i := findWait(nw.Waits, id)
if i < 0 {
return nw, fmt.Errorf("no open wait %s for %s", id, name)
}
nw.Waits[i].Watcher = watcher
return nw, nil
})
if err != nil {
return nil, err
}
if err := writeSessionWaits(p, store, session, now); err != nil {
return nil, fmt.Errorf("the wait was watched, but its file could not be written: %w", err)
}
_ = writeNodeIndex(p, store, now)
return map[string]any{"id": id, "watcher": watcher}, nil
}
// DoneWait removes a wait from the open list: it ended, with outcome (1629/AC2).
func DoneWait(p Paths, session, by, id, outcome string, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(session)
if err != nil {
return nil, err
}
name, err := callerName(p, by)
if err != nil {
return nil, err
}
id, outcome = strings.TrimSpace(id), strings.TrimSpace(outcome)
if id == "" {
return nil, errors.New("id is required: the one add answered")
}
key := waitsKey(p, name)
_, err = updateNamedWaits(store, key, func(nw NamedWaits) (NamedWaits, error) {
if err := bindCaller(&nw, name, p.Node, by, session, now); err != nil {
return nw, err
}
i := findWait(nw.Waits, id)
if i < 0 {
return nw, fmt.Errorf("no open wait %s for %s", id, name)
}
nw.Waits = append(append([]Wait(nil), nw.Waits[:i]...), nw.Waits[i+1:]...)
return nw, nil
})
if err != nil {
return nil, err
}
if err := writeSessionWaits(p, store, session, now); err != nil {
return nil, fmt.Errorf("the wait was done, but its file could not be written: %w", err)
}
_ = writeNodeIndex(p, store, now)
say("wait %s for %s (session %s) done: %s", id, name, session, outcome)
return map[string]any{"id": id, "done": true, "outcome": outcome}, nil
}
// SaidWait marks a wait past its bound as told to the operator: refused while it is still within its bound
// (1629/AC2).
func SaidWait(p Paths, session, by, id string, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(session)
if err != nil {
return nil, err
}
name, err := callerName(p, by)
if err != nil {
return nil, err
}
id = strings.TrimSpace(id)
if id == "" {
return nil, errors.New("id is required: the one add answered")
}
key := waitsKey(p, name)
_, err = updateNamedWaits(store, key, func(nw NamedWaits) (NamedWaits, error) {
if err := bindCaller(&nw, name, p.Node, by, session, now); err != nil {
return nw, err
}
i := findWait(nw.Waits, id)
if i < 0 {
return nw, fmt.Errorf("no open wait %s for %s", id, name)
}
if !nw.Waits[i].PastBound(now) {
return nw, fmt.Errorf("wait %s is not past its bound yet: said is for telling the operator about one that is", id)
}
nw.Waits[i].Said = true
return nw, nil
})
if err != nil {
return nil, err
}
if err := writeSessionWaits(p, store, session, now); err != nil {
return nil, fmt.Errorf("the wait was said, but its file could not be written: %w", err)
}
_ = writeNodeIndex(p, store, now)
return map[string]any{"id": id, "said": true}, nil
}
// ListWaits answers a name's open waits, with their age and whether they are past their bound (1629/AC1),
// and who currently holds the name. Read-only (review round 2, item c): unlike every other op it never
// claims an unbound name, and never refuses a session that is not (or not yet) the one bound to it — a
// session may want to look before adopting.
func ListWaits(p Paths, session, by string, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(session)
if err != nil {
return nil, err
}
name, err := callerName(p, by)
if err != nil {
return nil, err
}
nw, _, _, err := readNamedWaits(store, waitsKey(p, name))
if err != nil {
return nil, fmt.Errorf("the wait list could not be read: %w", err)
}
out := []map[string]any{}
for _, w := range nw.Waits {
entry := map[string]any{"id": w.ID, "what": w.What, "by": w.By, "address": w.Read.Address,
"bound_minutes": w.Bound, "at": w.At.Format(time.RFC3339), "age_seconds": int(now.Sub(w.At).Seconds()),
"past_bound": w.PastBound(now), "watcher": w.Watcher, "said": w.Said}
if len(w.Read.Arguments) > 0 {
entry["arguments"] = w.Read.Arguments
}
if w.Family != "" {
entry["family"] = w.Family
}
out = append(out, entry)
}
answer := map[string]any{"session": session, "name": name, "open": out}
if nw.CurrentSession != "" {
answer["held-by"] = map[string]any{"session": nw.CurrentSession, "last-called": nw.LastCall.UTC().Format(time.RFC3339)}
}
return answer, nil
}
// AdoptName binds name to session, always taking it over from whoever held it (novox/hq ADR 0490 §3): the
// mirrored files of both the session it is taken from and the one adopting it are rewritten at once, so a
// session that no longer owns a name never finds its old waits in its own file, and the new one does right
// away — no waiting on a bound, unlike the Stop and SessionStart hooks' own.
func AdoptName(p Paths, by, session string, store WaitStore, now time.Time) (map[string]any, error) {
session, err := requireSessionID(session)
if err != nil {
return nil, err
}
name, err := callerName(p, by)
if err != nil {
return nil, err
}
key := waitsKey(p, name)
var oldSession string
var oldLastCall time.Time
nw, err := updateNamedWaits(store, key, func(nw NamedWaits) (NamedWaits, error) {
oldSession, oldLastCall = nw.CurrentSession, nw.LastCall
nw.Name, nw.Node, nw.CurrentSession, nw.LastCall = name, p.Node, session, now.UTC()
return nw, nil
})
if err != nil {
return nil, err
}
tookOver := oldSession != "" && oldSession != session
if tookOver {
// oldSession's file is rewritten from the union of whatever names remain bound to it (ADR 0490 §1,
// review round 2 item 2): name no longer is one of them, but another might still be — never simply
// emptied.
if err := writeSessionWaits(p, store, oldSession, now); err != nil {
return nil, fmt.Errorf("%s was adopted, but the session it was taken from could not be told: %w", name, err)
}
}
if err := writeSessionWaits(p, store, session, now); err != nil {
return nil, fmt.Errorf("the name was adopted, but its file could not be written: %w", err)
}
if err := writeNodeIndex(p, store, now); err != nil {
say("writing the node-wide wait index after %s was adopted: %v", name, err)
}
// Review round 2, item b: a fresh three-strikes count starts under the new binding, on both ids.
waitsDir := WaitsDirPath(p)
_ = writeRefusalCounters(waitsDir, session, map[string]int{})
if tookOver {
_ = writeRefusalCounters(waitsDir, oldSession, map[string]int{})
}
answer := map[string]any{"name": name, "session": session, "open": len(nw.Waits)}
if tookOver {
answer["took-over-from"] = map[string]any{"session": oldSession, "last-called": oldLastCall.UTC().Format(time.RFC3339)}
say("%s adopted by session %s, taken over from %s (last called %s)", name, session, oldSession,
oldLastCall.UTC().Format(time.RFC3339))
} else {
say("%s adopted by session %s", name, session)
}
return answer, nil
}
// waitsFileIn is where one session's mirrored waits are, given a hook's own waits-dir argument: the hooks
// have no launch environment, only their command line, so they never recompute this from Paths.
func waitsFileIn(waitsDir, session string) string { return filepath.Join(waitsDir, session+".json") }
// nodeIndexPathIn is the node-wide index file, given a hook's own waits-dir argument.
func nodeIndexPathIn(waitsDir string) string { return filepath.Join(waitsDir, "_index.json") }
// readWaitsFile reads one session's mirrored file, as the hooks do: a plain local read, as the session's own
// account (never through fsys or the module's state), and never calling back into this module.
func readWaitsFile(path string) (WaitsFile, error) {
raw, err := os.ReadFile(path)
if err != nil {
return WaitsFile{}, err
}
var f WaitsFile
if err := json.Unmarshal(raw, &f); err != nil {
return WaitsFile{}, err
}
return f, nil
}
// readNodeIndex reads the node-wide index, as the SessionStart hook does: a plain local read, never through
// fsys or the module's state, and never calling back into this module.
func readNodeIndex(path string) (NodeIndex, error) {
raw, err := os.ReadFile(path)
if err != nil {
return NodeIndex{}, err
}
var idx NodeIndex
if err := json.Unmarshal(raw, &idx); err != nil {
return NodeIndex{}, err
}
return idx, nil
}
// waitTools is claude_code_wait: add, watched, done, said, list and adopt, over a name's wait list (novox/hq
// ADR 0489, ADR 0490).
func waitTools(p Paths) []stdio.Tool {
notLaunched := errors.New("this module was not launched with the bus; no wait can be kept")
return []stdio.Tool{
{Name: "claude_code_wait",
Description: "Your wait list, kept under your session's name (novox/hq ADR 0489, ADR 0490): what you " +
"still expect an answer to — an Approve, a delivery's end, anything you started and must act on when " +
"it ends. The plugin's Stop hook refuses to end your turn while a wait here is open and nothing " +
"follows it. op is add, watched, done, said, list or adopt; session (your current session id, named " +
"by the SessionStart hook) and by (your session's name, as who@node or who) are always required. A " +
"name is bound to whichever session last called for it; if your own id changed (a /clear) you must " +
"adopt your name again before it answers to you — the SessionStart hook lists every name on this " +
"node with an open wait and how. adopt takes the name over at once, however recently another " +
"session called for it; that session is told so, naming yours, the next time it calls. add takes " +
"what (what you are waiting for), read (the verb that answers where it stands: {address, " +
"arguments}), family (optional, the event mesh_wait can wait on) and bound_minutes (1 to 1440), and " +
"answers an id; at most 50 waits are open per name. watched takes id and watcher (what follows it: " +
"mesh_wait on its family, or a watcher agent you started). done takes id and outcome, and removes it " +
"from the open list. said takes id and marks a wait past its bound as told to the operator — " +
"refused while it is still within its bound. list answers the open waits, with their age and " +
"whether each is past its bound.",
Input: map[string]any{
"op": str("add, watched, done, said, list or adopt"),
"session": str("required: your session's current id"),
"by": str("required: your session's name, as who@node or who"),
"what": str("add: what you are waiting for, in plain words"),
"read": map[string]any{"type": "object", "description": "add: the verb that reads where this wait stands",
"properties": map[string]any{
"address": str("the address, as mesh_call takes it"),
"arguments": map[string]any{"type": "object", "description": "its arguments"},
}},
"family": str("add: the event family mesh_wait can wait on, when there is one"),
"bound_minutes": map[string]any{"type": "integer", "description": "add: minutes until this wait's bound, 1 to 1440"},
"id": str("watched, done or said: the id add answered"),
"watcher": str("watched: what follows this wait"),
"outcome": str("done: what the wait ended with"),
},
Run: func(a map[string]any) (any, error) {
if waiting == nil {
return nil, notLaunched
}
session, by := strArg(a, "session"), strArg(a, "by")
now := time.Now()
if waiting.Now != nil {
now = waiting.Now()
}
switch op := strArg(a, "op"); op {
case "add":
read := WaitRead{}
if m, ok := a["read"].(map[string]any); ok {
read.Address = strArg(m, "address")
if args, ok := m["arguments"].(map[string]any); ok {
read.Arguments = args
}
}
bound := 0
if n, ok := a["bound_minutes"].(float64); ok {
bound = int(n)
}
return AddWait(p, WaitInput{Session: session, By: by, What: strArg(a, "what"), Read: read,
Family: strArg(a, "family"), Bound: bound}, waiting.Store, now)
case "watched":
return WatchedWait(p, session, by, strArg(a, "id"), strArg(a, "watcher"), waiting.Store, now)
case "done":
return DoneWait(p, session, by, strArg(a, "id"), strArg(a, "outcome"), waiting.Store, now)
case "said":
return SaidWait(p, session, by, strArg(a, "id"), waiting.Store, now)
case "list":
return ListWaits(p, session, by, waiting.Store, now)
case "adopt":
return AdoptName(p, by, session, waiting.Store, now)
default:
return nil, fmt.Errorf("op is add, watched, done, said, list or adopt, not %q", op)
}
}},
}
}
@@ -0,0 +1,577 @@
package main
// A session's wait list (novox/hq ADR 0489, ADR 0490, issues 1629, 1744).
import (
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// memWaits is the `waits` state: values with revisions, create-only and compare-and-set, as the bus keeps it.
type memWaits struct {
mu sync.Mutex
m map[string]json.RawMessage
rev map[string]uint64
n uint64
}
func newMemWaits() *memWaits {
return &memWaits{m: map[string]json.RawMessage{}, rev: map[string]uint64{}}
}
func (s *memWaits) Get(key string) (json.RawMessage, uint64, bool, error) {
s.mu.Lock()
defer s.mu.Unlock()
raw, ok := s.m[key]
return raw, s.rev[key], ok, nil
}
func (s *memWaits) put(key string, value any) error {
raw, err := json.Marshal(value)
s.n++
s.m[key], s.rev[key] = raw, s.n
return err
}
func (s *memWaits) Create(key string, value any) error {
s.mu.Lock()
defer s.mu.Unlock()
if _, ok := s.m[key]; ok {
return stdio.ErrChanged
}
return s.put(key, value)
}
func (s *memWaits) Update(key string, value any, revision uint64) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.rev[key] != revision {
return stdio.ErrChanged
}
return s.put(key, value)
}
func (s *memWaits) Keys() ([]string, error) {
s.mu.Lock()
defer s.mu.Unlock()
out := make([]string, 0, len(s.m))
for k := range s.m {
out = append(out, k)
}
return out, nil
}
// addApproveWait adds the wait fixture the operator's Approve is kept under, for name on session.
func addApproveWait(t *testing.T, p Paths, store WaitStore, name, session string, at time.Time) string {
t.Helper()
out, err := AddWait(p, WaitInput{Session: session, By: name + "@" + p.Node, What: "the operator's Approve of pull request 42",
Read: WaitRead{Address: "mesh-delivery.recheck", Arguments: map[string]any{"pr": 42}}, Family: "mesh-delivery.decided",
Bound: 10}, store, at)
if err != nil {
t.Fatalf("add: %v", err)
}
return out["id"].(string)
}
// 1629/AC1: add answers an id; list answers the open waits with their age and whether each is past its bound.
func TestAddAnswersAnIDAndListShowsItsAgeAndPastBound(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
id := addApproveWait(t, p, store, "fix-1629", "sess-1", t0)
if !strings.HasPrefix(id, WaitPrefix) {
t.Fatalf("id %q does not start with %q", id, WaitPrefix)
}
out, err := ListWaits(p, "sess-1", "fix-1629@g14", store, t0.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
open := out["open"].([]map[string]any)
if len(open) != 1 || open[0]["id"] != id || open[0]["past_bound"] != false || open[0]["age_seconds"] != 60 {
t.Fatalf("%+v", open)
}
out, err = ListWaits(p, "sess-1", "fix-1629@g14", store, t0.Add(11*time.Minute))
if err != nil {
t.Fatal(err)
}
open = out["open"].([]map[string]any)
if open[0]["past_bound"] != true {
t.Fatalf("past its 10-minute bound after 11 minutes: %+v", open[0])
}
}
// 1629/AC2: watched, done and said change the wait; said is refused for a wait not past its bound; done removes
// it from the open list.
func TestWatchedDoneAndSaidChangeTheWait(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
id := addApproveWait(t, p, store, "fix-1629", "sess-2", t0)
if _, err := SaidWait(p, "sess-2", by, id, store, t0.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "not past its bound") {
t.Fatalf("said before the bound: %v", err)
}
if _, err := WatchedWait(p, "sess-2", by, id, "mesh_wait on mesh-delivery.decided", store, t0.Add(time.Minute)); err != nil {
t.Fatal(err)
}
out, _ := ListWaits(p, "sess-2", by, store, t0.Add(2*time.Minute))
open := out["open"].([]map[string]any)
if open[0]["watcher"] != "mesh_wait on mesh-delivery.decided" {
t.Fatalf("watcher not kept: %+v", open[0])
}
if _, err := SaidWait(p, "sess-2", by, id, store, t0.Add(11*time.Minute)); err != nil {
t.Fatalf("said past the bound: %v", err)
}
out, _ = ListWaits(p, "sess-2", by, store, t0.Add(12*time.Minute))
open = out["open"].([]map[string]any)
if open[0]["said"] != true {
t.Fatalf("said not kept: %+v", open[0])
}
if _, err := DoneWait(p, "sess-2", by, id, "approved", store, t0.Add(13*time.Minute)); err != nil {
t.Fatal(err)
}
out, _ = ListWaits(p, "sess-2", by, store, t0.Add(14*time.Minute))
if len(out["open"].([]map[string]any)) != 0 {
t.Fatalf("done did not remove it from the open list: %v", out)
}
if _, err := DoneWait(p, "sess-2", by, id, "approved", store, t0.Add(15*time.Minute)); err == nil {
t.Fatal("done on an already-done wait was not refused")
}
}
// 1629/AC3: every change rewrites the session's waits file whole, atomically; it holds exactly the open waits
// and the write time.
func TestEveryChangeRewritesTheWaitsFileWhole(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
id1 := addApproveWait(t, p, store, "fix-1629", "sess-3", t0)
f, err := readWaitsFile(waitsFilePath(p, "sess-3"))
if err != nil {
t.Fatal(err)
}
if f.Session != "sess-3" || len(f.Waits) != 1 || f.Waits[0].ID != id1 || !f.At.Equal(t0.UTC()) {
t.Fatalf("%+v", f)
}
// Written whole: no .mesh-tmp left beside it.
if _, err := os.Stat(waitsFilePath(p, "sess-3") + ".mesh-tmp"); !os.IsNotExist(err) {
t.Fatalf("a temp file was left beside it: %v", err)
}
id2, err := AddWait(p, WaitInput{Session: "sess-3", By: by, What: "a second wait",
Read: WaitRead{Address: "mesh-delivery.recheck"}, Bound: 5}, store, t0.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
f, err = readWaitsFile(waitsFilePath(p, "sess-3"))
if err != nil || len(f.Waits) != 2 || !f.At.Equal(t0.Add(time.Minute).UTC()) {
t.Fatalf("%+v %v", f, err)
}
if _, err := DoneWait(p, "sess-3", by, id1, "approved", store, t0.Add(2*time.Minute)); err != nil {
t.Fatal(err)
}
f, err = readWaitsFile(waitsFilePath(p, "sess-3"))
if err != nil || len(f.Waits) != 1 || f.Waits[0].ID != id2["id"] {
t.Fatalf("the file does not hold exactly the open waits: %+v %v", f, err)
}
}
// 1629/AC4: the waits are read again after the module "restarts" (the state is kept) and for the same name
// after a /clear that keeps calling with the same session id; a different name lists none of them.
func TestWaitsSurviveARestartAndAreNeverSeenByAnotherName(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
addApproveWait(t, p, store, "fix-1629", "sess-4", t0)
// A restart: a fresh Waiting over the same store reads the same waits.
restarted := &Waiting{Store: store, Now: func() time.Time { return t0.Add(time.Minute) }}
_ = restarted
out, err := ListWaits(p, "sess-4", by, store, t0.Add(time.Minute))
if err != nil || len(out["open"].([]map[string]any)) != 1 {
t.Fatalf("not read again after a restart: %v %v", out, err)
}
// The same session id, called again: still lists it.
out, err = ListWaits(p, "sess-4", by, store, t0.Add(2*time.Minute))
if err != nil || len(out["open"].([]map[string]any)) != 1 {
t.Fatalf("not read again for the same name: %v %v", out, err)
}
// A different name, same node: none of them.
out, err = ListWaits(p, "sess-other", "fix-other@g14", store, t0.Add(2*time.Minute))
if err != nil || len(out["open"].([]map[string]any)) != 0 {
t.Fatalf("another name saw them: %v %v", out, err)
}
}
// 1629/AC5: add refuses an empty what, a bound under 1 or over 1440 minutes, a read address that is not an
// address, and more than 50 open waits per name, naming the argument.
func TestAddRefusesOutsideItsBounds(t *testing.T) {
p, _ := node(t, "g14")
base := func() WaitInput {
return WaitInput{Session: "sess-5", By: "fix-1629@" + p.Node, What: "something", Read: WaitRead{Address: "mesh-delivery.recheck"}, Bound: 10}
}
for name, change := range map[string]struct {
mutate func(*WaitInput)
want string
}{
"empty what": {func(in *WaitInput) { in.What = "" }, "what is required"},
"bound zero": {func(in *WaitInput) { in.Bound = 0 }, "bound_minutes"},
"bound under 1": {func(in *WaitInput) { in.Bound = 0 }, "bound_minutes"},
"bound over 1440": {func(in *WaitInput) { in.Bound = 1441 }, "bound_minutes"},
"empty address": {func(in *WaitInput) { in.Read.Address = "" }, "read.address"},
"address no dot": {func(in *WaitInput) { in.Read.Address = "notanaddress" }, "read.address"},
"address has spaces": {func(in *WaitInput) { in.Read.Address = "not an address" }, "read.address"},
} {
in := base()
change.mutate(&in)
store := newMemWaits()
if _, err := AddWait(p, in, store, t0); err == nil || !strings.Contains(err.Error(), change.want) {
t.Errorf("%s: %v", name, err)
}
}
// More than 50 open waits per name.
store := newMemWaits()
for i := 0; i < MostWaits; i++ {
if _, err := AddWait(p, base(), store, t0); err != nil {
t.Fatalf("wait %d: %v", i, err)
}
}
if _, err := AddWait(p, base(), store, t0); err == nil || !strings.Contains(err.Error(), "at most 50 open waits") {
t.Fatalf("the 51st wait was not refused: %v", err)
}
}
// Review round 1 item (b): list, watched, done and said refuse a session id that is not a plain one (the
// same waitSessionID add already holds to), naming "session" — not silently treating it as just-another-key.
func TestListWatchedDoneAndSaidRefuseABadSessionID(t *testing.T) {
p, _ := node(t, "g14")
by := "fix-1629@g14"
for _, bad := range []string{"", "a/b", "../x"} {
store := newMemWaits()
if _, err := ListWaits(p, bad, by, store, t0); err == nil || !strings.Contains(err.Error(), "session") {
t.Errorf("list %q: %v", bad, err)
}
if _, err := WatchedWait(p, bad, by, "w-1", "watcher", store, t0); err == nil || !strings.Contains(err.Error(), "session") {
t.Errorf("watched %q: %v", bad, err)
}
if _, err := DoneWait(p, bad, by, "w-1", "outcome", store, t0); err == nil || !strings.Contains(err.Error(), "session") {
t.Errorf("done %q: %v", bad, err)
}
if _, err := SaidWait(p, bad, by, "w-1", store, t0); err == nil || !strings.Contains(err.Error(), "session") {
t.Errorf("said %q: %v", bad, err)
}
}
}
// 1629/AC6: the tool is served through the module's own dispatch.
func TestTheWaitToolIsServedThroughTheModulesDispatch(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
t.Cleanup(func() { waiting = nil })
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "sess-6", "by": "fix-1629@g14",
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
if isErr {
t.Fatalf("add: %s", text)
}
var added map[string]any
if err := json.Unmarshal([]byte(text), &added); err != nil {
t.Fatal(err)
}
id, _ := added["id"].(string)
if id == "" {
t.Fatalf("no id answered: %s", text)
}
text, isErr = s.call(t, "claude_code_wait", map[string]any{"op": "list", "session": "sess-6", "by": "fix-1629@g14"})
if isErr || !strings.Contains(text, id) {
t.Fatalf("list: %v %s", isErr, text)
}
text, isErr = s.call(t, "claude_code_wait", map[string]any{"op": "done", "session": "sess-6", "by": "fix-1629@g14",
"id": id, "outcome": "approved"})
if isErr {
t.Fatalf("done: %s", text)
}
}
// ---- ADR 0490: a name's waits survive a session's id changing, by adopting -----------------------------
// ADR 0490 §1: a name nothing has bound yet is claimed by the first session that calls for it (add), without
// a separate adopt first.
func TestAFirstAddBindsAnUnboundNameAtOnce(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
id := addApproveWait(t, p, store, "fix-1629", "sess-a", t0)
out, err := ListWaits(p, "sess-a", "fix-1629@g14", store, t0)
if err != nil || len(out["open"].([]map[string]any)) != 1 || out["open"].([]map[string]any)[0]["id"] != id {
t.Fatalf("%v %v", out, err)
}
}
// ADR 0490 §1/§3: once a name is bound to a session, a different, non-adopted session calling for the same
// name (any op) is refused, naming the session that holds it and when it last called.
func TestADifferentSessionIsRefusedWithoutAdopting(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
addApproveWait(t, p, store, "fix-1629", "sess-old", t0)
if _, err := AddWait(p, WaitInput{Session: "sess-new", By: by, What: "another", Read: WaitRead{Address: "mesh-delivery.recheck"},
Bound: 10}, store, t0.Add(time.Minute)); err == nil || !strings.Contains(err.Error(), "sess-old") {
t.Fatalf("add from a different session was not refused: %v", err)
}
// list, read-only (review round 2 item c), never refuses: it just says who holds the name.
out, err := ListWaits(p, "sess-new", by, store, t0.Add(time.Minute))
if err != nil {
t.Fatalf("list from a different session was refused: %v", err)
}
held, _ := out["held-by"].(map[string]any)
if held == nil || held["session"] != "sess-old" {
t.Fatalf("list did not say who holds the name: %v", out)
}
}
// ADR 0490 §2/§3: adopt right after a simulated /clear binds the name to the new session id at once; the
// mirrored file of the old session id is emptied and the new one holds the waits, with no delay.
func TestAdoptRightAfterAClearBindsAtOnce(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
id := addApproveWait(t, p, store, "fix-1629", "sess-old", t0)
out, err := AdoptName(p, by, "sess-new", store, t0.Add(time.Minute))
if err != nil {
t.Fatal(err)
}
if out["session"] != "sess-new" || out["open"] != 1 {
t.Fatalf("%v", out)
}
took, _ := out["took-over-from"].(map[string]any)
if took == nil || took["session"] != "sess-old" {
t.Fatalf("the answer did not name the earlier id: %v", out)
}
oldFile, err := readWaitsFile(waitsFilePath(p, "sess-old"))
if err != nil || len(oldFile.Waits) != 0 {
t.Fatalf("the old session's file still holds waits: %+v %v", oldFile, err)
}
newFile, err := readWaitsFile(waitsFilePath(p, "sess-new"))
if err != nil || len(newFile.Waits) != 1 || newFile.Waits[0].ID != id {
t.Fatalf("the new session's file does not hold the waits: %+v %v", newFile, err)
}
// The new session can act on the name at once: no 10-minute rule.
if _, err := ListWaits(p, "sess-new", by, store, t0.Add(time.Minute)); err != nil {
t.Fatalf("the new session could not act right after adopting: %v", err)
}
}
// ADR 0490 §3: adopt of a name bound to another id always takes it over, however recently that id called —
// no bound protects it.
func TestAdoptAlwaysTakesOverEvenJustCalled(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
addApproveWait(t, p, store, "fix-1629", "sess-fresh", t0)
// sess-fresh calls again right before being adopted away from.
if _, err := ListWaits(p, "sess-fresh", by, store, t0.Add(10*time.Second)); err != nil {
t.Fatal(err)
}
out, err := AdoptName(p, by, "sess-taker", store, t0.Add(11*time.Second))
if err != nil {
t.Fatalf("adopt was refused even though it always takes over: %v", err)
}
if out["session"] != "sess-taker" {
t.Fatalf("%v", out)
}
}
// ADR 0490 §3: the earlier id's next claude_code_wait call (any op) answers that its name now answers to
// another session, naming it.
func TestTheEarlierIDIsToldOnItsNextCall(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
addApproveWait(t, p, store, "fix-1629", "sess-old", t0)
if _, err := AdoptName(p, by, "sess-new", store, t0.Add(time.Minute)); err != nil {
t.Fatal(err)
}
_, err := AddWait(p, WaitInput{Session: "sess-old", By: by, What: "another", Read: WaitRead{Address: "mesh-delivery.recheck"},
Bound: 10}, store, t0.Add(2*time.Minute))
if err == nil || !strings.Contains(err.Error(), "sess-new") || !strings.Contains(err.Error(), "adopt") {
t.Fatalf("the earlier id's next call did not name the new session: %v", err)
}
}
// ADR 0490 §2: the node-wide index lists every name on this node with an open wait, for the SessionStart
// hook to read directly.
func TestTheNodeWideIndexListsEveryNameWithOpenWaits(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
addApproveWait(t, p, store, "fix-1629", "sess-x", t0)
addApproveWait(t, p, store, "fix-1744", "sess-y", t0)
if _, err := DoneWait(p, "sess-x", "fix-1629@g14", wantsOnlyWait(t, store, p, "fix-1629", "sess-x"), "done",
store, t0.Add(time.Minute)); err != nil {
t.Fatal(err)
}
idx, err := readNodeIndex(nodeIndexPath(p))
if err != nil {
t.Fatal(err)
}
if idx.Node != "g14" || len(idx.Names) != 1 || idx.Names[0].Name != "fix-1744" || idx.Names[0].Open != 1 ||
idx.Names[0].CurrentSession != "sess-y" {
t.Fatalf("%+v", idx)
}
}
// wantsOnlyWait is the id of the one wait a fixture added, read back through list.
func wantsOnlyWait(t *testing.T, store WaitStore, p Paths, name, session string) string {
t.Helper()
out, err := ListWaits(p, session, name+"@"+p.Node, store, t0)
if err != nil {
t.Fatal(err)
}
open := out["open"].([]map[string]any)
if len(open) != 1 {
t.Fatalf("%v", open)
}
return open[0]["id"].(string)
}
// claude_code_wait op=adopt, served through the module's own dispatch (1629/AC6, ADR 0490).
func TestAdoptIsServedThroughTheModulesDispatch(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
waiting = &Waiting{Store: store, Now: func() time.Time { return t0 }}
t.Cleanup(func() { waiting = nil })
s := serve(t, p, &bus{kept: map[string]map[string]any{}}, memConfig{})
text, isErr := s.call(t, "claude_code_wait", map[string]any{"op": "add", "session": "sess-old", "by": "fix-1629@g14",
"what": "the operator's Approve", "read": map[string]any{"address": "mesh-delivery.recheck"}, "bound_minutes": 10})
if isErr {
t.Fatalf("add: %s", text)
}
text, isErr = s.call(t, "claude_code_wait", map[string]any{"op": "adopt", "session": "sess-new", "by": "fix-1629@g14"})
if isErr || !strings.Contains(text, "sess-old") {
t.Fatalf("adopt: %v %s", isErr, text)
}
}
// ---- Review round 2: a session id's mirrored file is the union of every name bound to it -----------------
// Review round 2, item 2 (ADR 0490 §1): one session id with two different names bound to it holds both
// names' waits in its one mirrored file, not only the name an op last acted on.
func TestASessionsFileHoldsEveryNameBoundToIt(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
idX := addApproveWait(t, p, store, "fix-x", "sess-shared", t0)
idY := addApproveWait(t, p, store, "fix-y", "sess-shared", t0.Add(time.Minute))
f, err := readWaitsFile(waitsFilePath(p, "sess-shared"))
if err != nil {
t.Fatal(err)
}
ids := map[string]bool{}
for _, w := range f.Waits {
ids[w.ID] = true
}
if len(f.Waits) != 2 || !ids[idX] || !ids[idY] {
t.Fatalf("the session's file does not hold both names' waits: %+v", f)
}
}
// Review round 2, item 2: adopting one of two names bound to the same session away keeps that session's
// file holding the other name's waits — it is never emptied wholesale.
func TestAdoptingOneNameAwayKeepsTheSessionsOtherNamesWaits(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
idX := addApproveWait(t, p, store, "fix-x", "sess-shared", t0)
idY := addApproveWait(t, p, store, "fix-y", "sess-shared", t0.Add(time.Minute))
if _, err := AdoptName(p, "fix-y@g14", "sess-y-new", store, t0.Add(2*time.Minute)); err != nil {
t.Fatal(err)
}
sharedFile, err := readWaitsFile(waitsFilePath(p, "sess-shared"))
if err != nil || len(sharedFile.Waits) != 1 || sharedFile.Waits[0].ID != idX {
t.Fatalf("sess-shared should still hold fix-x's wait: %+v %v", sharedFile, err)
}
newFile, err := readWaitsFile(waitsFilePath(p, "sess-y-new"))
if err != nil || len(newFile.Waits) != 1 || newFile.Waits[0].ID != idY {
t.Fatalf("sess-y-new should hold fix-y's wait: %+v %v", newFile, err)
}
}
// Review round 2, item b: adopt clears the refusal counter of both the earlier id and the new one, so a
// fresh three-strikes count starts under the new binding.
func TestAdoptClearsTheRefusalCounterOfBothIDs(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
addApproveWait(t, p, store, "fix-1629", "sess-old", t0)
waitsDir := WaitsDirPath(p)
if err := writeRefusalCounters(waitsDir, "sess-old", map[string]int{"w-1": 2}); err != nil {
t.Fatal(err)
}
if err := writeRefusalCounters(waitsDir, "sess-new", map[string]int{"w-2": 3}); err != nil {
t.Fatal(err)
}
if _, err := AdoptName(p, by, "sess-new", store, t0.Add(time.Minute)); err != nil {
t.Fatal(err)
}
if c := readRefusalCounters(waitsDir, "sess-old"); len(c) != 0 {
t.Errorf("sess-old's counter was not cleared: %v", c)
}
if c := readRefusalCounters(waitsDir, "sess-new"); len(c) != 0 {
t.Errorf("sess-new's counter was not cleared: %v", c)
}
}
// Review round 2, item c: list is read-only — it never binds an unbound name, and never refuses a
// mismatched session; it answers the name's waits and who currently holds it.
func TestListNeverBindsAndAnswersWhoHoldsTheName(t *testing.T) {
p, _ := node(t, "g14")
store := newMemWaits()
by := "fix-1629@g14"
// An unbound name: list answers no waits, and binds nothing (a later add from a different session still
// succeeds, as the first caller for an unbound name).
out, err := ListWaits(p, "sess-reader", by, store, t0)
if err != nil || len(out["open"].([]map[string]any)) != 0 {
t.Fatalf("an unbound name: %v %v", out, err)
}
if _, found, _ := readSessionBinding(store, waitsKey(p, "fix-1629")); found {
t.Fatal("list bound an unbound name")
}
if _, err := AddWait(p, WaitInput{Session: "sess-first", By: by, What: "x", Read: WaitRead{Address: "mesh-delivery.recheck"},
Bound: 10}, store, t0); err != nil {
t.Fatalf("add after a read-only list: %v", err)
}
// A name bound to another session: list still answers (never refused), and says who holds it.
out, err = ListWaits(p, "sess-other", by, store, t0.Add(time.Minute))
if err != nil {
t.Fatalf("list for a name held by another session was refused: %v", err)
}
if len(out["open"].([]map[string]any)) != 1 {
t.Fatalf("list did not answer the name's waits: %v", out)
}
held, _ := out["held-by"].(map[string]any)
if held == nil || held["session"] != "sess-first" {
t.Fatalf("list did not say who holds the name: %v", out)
}
}
// readSessionBinding is a thin test helper reading the raw NamedWaits at key, without claiming it.
func readSessionBinding(store WaitStore, key string) (NamedWaits, bool, error) {
nw, _, found, err := readNamedWaits(store, key)
return nw, found, err
}
+3 -1
View File
@@ -28,7 +28,8 @@
"config",
"proposals",
"questions",
"memory"
"memory",
"waits"
],
"reads": [
"claude-licence-manager.bindings"
@@ -67,6 +68,7 @@
"claude_code_proposals",
"claude_code_ask",
"claude_code_answer",
"claude_code_wait",
"claude_code_settings_get",
"claude_code_settings_set",
"claude_code_settings_clear",
+3 -1
View File
@@ -24,7 +24,7 @@ machine's resources a seat, these tools become its verbs.
## Tools
All five only read.
All six only read.
| tool | what | replaces |
|---|---|---|
@@ -32,6 +32,7 @@ All five only read.
| `disk_top` | Over a window. `by=unit` (the default) ranks systemd units and containers by bytes moved, from their cgroups. `by=container` ranks containers by name, and `by=process` ranks processes | `iotop` |
| `disk_filesystems` | Each filesystem that stores data, said once however many places it is mounted: size, used and free GiB, used % as `df` counts it, inodes used % | `df -h`, `df -i` |
| `disk_devices` | Each disk without sampling: model, size, whether it spins, scheduler, queue limit, what it is built on, and since boot the GiB read and written, the busy share and the wait per read and per write | `lsblk`, `cat /proc/diskstats` |
| `disk_folders` | Which folders fill a disk, like `du -x`: the largest folders under `path` (default `/`) to `depth` levels (default 2, at most 3), `limit` of them (default 20, at most 100), with bytes and GiB. Sizes are allocated bytes, a hard link once. It stays on the path's one filesystem: a mount below it is named, never entered. It reads every folder the tool runner can read (docker's included) and no content; folders it cannot read are counted and named. After 25 s (the tool runner gives a call 30 s) it answers what it has measured, marked `partial`, with `stopped_at` (novox/hq issue 1804) | `du -x`, `ncdu` |
| `cpu_top` | Samples a window (`seconds`, default 3, at most 10). It answers the machine's CPU in use (% of all CPUs and as CPUs busy, from `/proc/stat`), then `by=unit` (the default) ranks systemd units and containers by CPU, from their cgroups' cpu.stat, busiest first (`limit`, default 15), with the busy time no unit accounts for; `by=container` ranks containers by name | `top`, `systemd-cgtop` |
### How each number is read
@@ -87,4 +88,5 @@ cover:
- processes ranked, one that started inside the window left out, unreadable ones counted;
- filesystems said once, mount escapes undone, a statfs that fails said;
- the window's bounds;
- folders ranked biggest first to the depth asked, a mount never entered, unreadable folders named, a walk past its bound answered as partial with where it stopped, a hard link counted once;
- the manifest: tools listed equal tools served, all read-only, no resources, no machine named.
+197
View File
@@ -0,0 +1,197 @@
package main
import (
"fmt"
"io/fs"
"math"
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"time"
)
// FolderBudget is how long a folder walk may take before it answers what it has measured.
const FolderBudget = 25 * time.Second
// namedUnreadable is how many unreadable folders are named; the rest are only counted.
const namedUnreadable = 50
// Folder is one folder's size: everything under it on the same filesystem.
type Folder struct {
Path string `json:"path"`
Depth int `json:"depth"`
Bytes int64 `json:"bytes"`
GiB float64 `json:"gib"`
}
// FolderReport is the answer of disk_folders.
type FolderReport struct {
Path string `json:"path"`
Bytes int64 `json:"bytes"`
GiB float64 `json:"gib"`
Largest []Folder `json:"largest"`
Partial bool `json:"partial"`
StoppedAt string `json:"stopped_at,omitempty"`
Unreadable int `json:"unreadable"`
UnreadableFolders []string `json:"unreadable_folders,omitempty"`
Skipped []string `json:"other_filesystems_skipped,omitempty"`
Said []string `json:"said"`
}
func (m *Machine) readDir(real string) ([]os.DirEntry, error) {
if m.ReadDir != nil {
return m.ReadDir(real)
}
return os.ReadDir(real)
}
func (m *Machine) now() time.Time {
if m.Clock != nil {
return m.Clock()
}
return time.Now()
}
func devOf(path string, info fs.FileInfo) uint64 {
if st, ok := info.Sys().(*syscall.Stat_t); ok {
return uint64(st.Dev)
}
return 0
}
type inodeKey struct{ dev, ino uint64 }
// walk is one folder measurement.
type walk struct {
m *Machine
maxDepth int
dev uint64
deadline time.Time
seen map[inodeKey]bool
folders []Folder
bad []string
skipped []string
stopped string
}
// Folders measures path like du -x: allocated bytes, a hard link once, never entering another filesystem.
func (m *Machine) Folders(path string, depth, limit int) (FolderReport, error) {
if !filepath.IsAbs(path) {
return FolderReport{}, fmt.Errorf("path must be absolute, not %q", path)
}
path = filepath.Clean(path)
info, err := os.Stat(m.path(path))
if err != nil {
return FolderReport{}, fmt.Errorf("path %s: %w", path, err)
}
if !info.IsDir() {
return FolderReport{}, fmt.Errorf("path %s is not a folder", path)
}
budget := m.Budget
if budget == 0 {
budget = FolderBudget
}
w := &walk{m: m, maxDepth: depth, deadline: m.now().Add(budget), seen: map[inodeKey]bool{}}
w.dev = w.device(path, info)
total := w.dir(path, 0)
sort.SliceStable(w.folders, func(i, j int) bool {
if w.folders[i].Bytes != w.folders[j].Bytes {
return w.folders[i].Bytes > w.folders[j].Bytes
}
return w.folders[i].Path < w.folders[j].Path
})
if len(w.folders) > limit {
w.folders = w.folders[:limit]
}
sort.Strings(w.bad)
named := w.bad
if len(named) > namedUnreadable {
named = named[:namedUnreadable]
}
r := FolderReport{
Path: path, Bytes: total, GiB: gibOf(total), Largest: w.folders,
Partial: w.stopped != "", StoppedAt: w.stopped,
Unreadable: len(w.bad), UnreadableFolders: named, Skipped: w.skipped,
}
if r.Largest == nil {
r.Largest = []Folder{}
}
var said []string
if r.Partial {
said = append(said, fmt.Sprintf("PARTIAL: the walk stopped after %s at %s; sizes are what was measured so far", budget, r.StoppedAt))
}
said = append(said, fmt.Sprintf("%s holds %.1f GiB on its own filesystem", path, r.GiB))
if r.Unreadable > 0 {
said = append(said, fmt.Sprintf("%d folders could not be read (%s), so their size is missing", r.Unreadable, strings.Join(named, ", ")))
}
if len(r.Skipped) > 0 {
said = append(said, "other filesystems not entered: "+strings.Join(r.Skipped, ", "))
}
r.Said = said
return r, nil
}
func (w *walk) device(path string, info fs.FileInfo) uint64 {
if w.m.DevOf != nil {
return w.m.DevOf(path, info)
}
return devOf(path, info)
}
// dir is the allocated bytes under a folder; it records the folder when it is within the depth.
func (w *walk) dir(path string, depth int) int64 {
if w.stopped != "" {
return 0
}
if w.m.now().After(w.deadline) {
w.stopped = path
return 0
}
entries, err := w.m.readDir(w.m.path(path))
if err != nil {
w.bad = append(w.bad, path)
return 0
}
var total int64
for _, e := range entries {
child := filepath.Join(path, e.Name())
info, err := os.Lstat(w.m.path(child))
if err != nil {
continue // gone since it was listed
}
if info.IsDir() {
if w.device(child, info) != w.dev {
w.skipped = append(w.skipped, child)
continue
}
total += w.allocated(info, false) + w.dir(child, depth+1)
continue
}
total += w.allocated(info, true)
}
if depth >= 1 && depth <= w.maxDepth {
w.folders = append(w.folders, Folder{Path: path, Depth: depth, Bytes: total, GiB: gibOf(total)})
}
return total
}
// allocated is the bytes an entry occupies on disk, as du counts them; a file with several links once.
func (w *walk) allocated(info fs.FileInfo, file bool) int64 {
st, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return info.Size()
}
if file && st.Nlink > 1 {
k := inodeKey{uint64(st.Dev), uint64(st.Ino)}
if w.seen[k] {
return 0
}
w.seen[k] = true
}
return st.Blocks * 512
}
func gibOf(b int64) float64 { return math.Round(float64(b)/gib*100) / 100 }
@@ -0,0 +1,170 @@
package main
import (
"fmt"
"io/fs"
"os"
"strings"
"testing"
"time"
)
const mb = 1 << 20
// megabytes writes a file of n MiB under the fake root.
func (f *fake) megabytes(path string, n int) { f.file(path, strings.Repeat("x", n*mb)) }
func aFullDisk(t *testing.T) *fake {
f := newFake(t)
f.megabytes("var/lib/docker/overlay2/a/layer", 8)
f.megabytes("var/lib/docker/overlay2/b/layer", 4)
f.megabytes("var/log/big.log", 3)
f.megabytes("home/jo/video.mkv", 6)
f.megabytes("home/jo/deep/er/est/file", 1)
f.dir("empty")
return f
}
func folders(t *testing.T, m *Machine, args map[string]any) FolderReport {
t.Helper()
out, err := tool(t, m, "disk_folders").Run(args)
if err != nil {
t.Fatal(err)
}
return out.(FolderReport)
}
func paths(r FolderReport) string {
var p []string
for _, f := range r.Largest {
p = append(p, f.Path)
}
return strings.Join(p, ",")
}
// 1804/AC1: given a disk with folders of known sizes, when disk_folders runs with the defaults, then it answers
// the largest folders to depth 2, biggest first, with bytes and GiB, and the total of the path.
func TestFoldersAreRankedBiggestFirstToDepthTwoByDefault(t *testing.T) {
r := folders(t, aFullDisk(t).machine(), nil)
if got := paths(r); !strings.HasPrefix(got, "/var,/var/lib,") || !strings.Contains(got, "/home,/home/jo") {
t.Fatalf("ranked %s", got)
}
if r.Path != "/" || r.Partial || r.Unreadable != 0 {
t.Errorf("%+v", r)
}
top := r.Largest[0]
if top.Path != "/var" || top.Depth != 1 || top.Bytes < 15*mb || top.Bytes > 16*mb || top.GiB <= 0 {
t.Errorf("top %+v", top)
}
for _, f := range r.Largest {
if f.Depth > 2 || f.Depth < 1 {
t.Errorf("%+v is outside depth 2", f)
}
if strings.Contains(f.Path, "overlay2") {
t.Errorf("%s is deeper than 2", f.Path)
}
}
if r.Bytes < 22*mb || r.Bytes > 24*mb {
t.Errorf("total %d", r.Bytes)
}
}
// 1804/AC2: given a path, a depth and a limit, then only that folder is measured, depth is held at 3 and limit at 100.
func TestFoldersTakeAPathADepthAndALimitHeldAtThreeAndAHundred(t *testing.T) {
f := aFullDisk(t)
r := folders(t, f.machine(), map[string]any{"path": "/var/lib/docker", "depth": 1.0})
if r.Path != "/var/lib/docker" || paths(r) != "/var/lib/docker/overlay2" {
t.Errorf("%s: %s", r.Path, paths(r))
}
r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0, "limit": 2.0})
if len(r.Largest) != 2 {
t.Errorf("limit 2 gave %d", len(r.Largest))
}
r = folders(t, f.machine(), map[string]any{"path": "/home", "depth": 9.0})
if got := paths(r); !strings.Contains(got, "/home/jo/deep/er") || strings.Contains(got, "/home/jo/deep/er/est") {
t.Errorf("depth 3 under /home is /home/jo/deep/er: %s", got)
}
tl := tool(t, f.machine(), "disk_folders")
for _, bad := range []map[string]any{{"depth": 0.0}, {"limit": 0.0}, {"depth": "deep"}, {"path": "relative/dir"}, {"path": "/nope"}, {"path": "/var/log/big.log"}} {
if _, err := tl.Run(bad); err == nil {
t.Errorf("%v was accepted", bad)
}
}
for i := 0; i < 120; i++ {
f.file(fmt.Sprintf("many/d%03d/f", i), "x")
}
if r := folders(t, f.machine(), map[string]any{"path": "/many", "limit": 500.0}); len(r.Largest) != 100 {
t.Errorf("limit held at %d, not 100", len(r.Largest))
}
}
// 1804/AC3: given a mount below the path, then it is never entered or counted, and is named.
func TestFoldersNeverCrossIntoAnotherFilesystem(t *testing.T) {
f := aFullDisk(t)
m := f.machine()
m.DevOf = func(path string, info fs.FileInfo) uint64 {
if path == "/var/lib/docker" {
return 7
}
return 1
}
r := folders(t, m, nil)
if strings.Contains(paths(r), "docker") || len(r.Skipped) != 1 || r.Skipped[0] != "/var/lib/docker" {
t.Errorf("%s, skipped %v", paths(r), r.Skipped)
}
if r.Bytes > 14*mb {
t.Errorf("total %d includes the mount", r.Bytes)
}
}
// 1804/AC4: given folders it cannot read, then they are counted and named and the rest is still answered.
func TestUnreadableFoldersAreCountedAndNamed(t *testing.T) {
f := aFullDisk(t)
m := f.machine()
real := os.ReadDir
m.ReadDir = func(p string) ([]os.DirEntry, error) {
if strings.HasSuffix(p, "/var/log") || strings.HasSuffix(p, "/overlay2/b") {
return nil, fmt.Errorf("open %s: permission denied", p)
}
return real(p)
}
r := folders(t, m, map[string]any{"depth": 3.0})
if r.Unreadable != 2 || strings.Join(r.UnreadableFolders, ",") != "/var/lib/docker/overlay2/b,/var/log" {
t.Errorf("%d: %v", r.Unreadable, r.UnreadableFolders)
}
if r.Partial || !strings.Contains(strings.Join(r.Said, "\n"), "2 folders could not be read") {
t.Errorf("%+v", r)
}
}
// 1804/AC5: given a walk that outlasts its bound, then it answers what it measured, partial, with where it stopped.
func TestALongWalkAnswersPartialAndSaysWhereItStopped(t *testing.T) {
f := aFullDisk(t)
m := f.machine()
now := time.Unix(0, 0)
m.Clock = func() time.Time { now = now.Add(10 * time.Second); return now }
r := folders(t, m, nil)
if !r.Partial || r.StoppedAt == "" || len(r.Said) < 2 || !strings.HasPrefix(r.Said[0], "PARTIAL") || !strings.Contains(r.Said[0], r.StoppedAt) {
t.Errorf("%+v", r)
}
}
// 1804/AC6: a hard link is counted once, and the tool changes nothing.
func TestAHardLinkIsCountedOnce(t *testing.T) {
f := newFake(t)
f.megabytes("a/big", 5)
if err := os.Link(f.root+"/a/big", f.root+"/a/again"); err != nil {
t.Skip(err)
}
r := folders(t, f.machine(), nil)
if r.Bytes > 6*mb {
t.Errorf("counted twice: %d", r.Bytes)
}
}
// 1804/AC7: the bound sits below the 30 s the tool runner gives a call, so a partial answer reaches its caller.
func TestTheWalkBoundIsBelowTheToolRunnersCallLimit(t *testing.T) {
if FolderBudget >= 30*time.Second {
t.Errorf("the budget %s is not below the 30 s a call gets", FolderBudget)
}
}
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
@@ -53,6 +54,13 @@ type Machine struct {
Run Runner
Wait func(time.Duration)
Statfs func(path string) (FSStat, error)
// The folder walk's seams, all optional: the folder reader, the filesystem a path is on, the clock and
// the time the walk may take (60 s when zero).
ReadDir func(path string) ([]os.DirEntry, error)
DevOf func(path string, info fs.FileInfo) uint64
Clock func() time.Time
Budget time.Duration
}
// Here is the machine this process runs on.
+28
View File
@@ -136,6 +136,34 @@ func Tools(m *Machine) []stdio.Tool {
}, nil
},
},
{
Name: "disk_folders",
Description: "Which folders fill a disk, like du -x: the largest folders under a path, measured on disk (allocated bytes, a hard link " +
"once), staying on the path's one filesystem (a mount below it is named, never entered). path defaults to /, depth to 2 (at most 3), " +
"limit to 20 (at most 100). Sizes in bytes and GiB. It reads every folder the tool runner can read, none of the content; folders " +
"it cannot read are counted and named. It stops after 25 s and answers what it measured so far, marked partial, with where it stopped. " +
"Replaces du. (r)",
Input: map[string]any{
"path": map[string]any{"type": "string", "description": "the absolute folder to measure (default /)"},
"depth": map[string]any{"type": "integer", "description": "how many levels below path to rank (default 2, at most 3)"},
"limit": map[string]any{"type": "integer", "description": "how many folders to answer (default 20, at most 100)"},
},
Run: func(args map[string]any) (any, error) {
depth, err := count(args, "depth", 2, 1, 3)
if err != nil {
return nil, err
}
limit, err := count(args, "limit", 20, 1, 100)
if err != nil {
return nil, err
}
path := text(args, "path")
if path == "" {
path = "/"
}
return m.Folders(path, depth, limit)
},
},
{
Name: "disk_filesystems",
Description: "The space of every filesystem that stores data (not proc, tmpfs or overlay layers): size, used and free in GiB, " +
+1
View File
@@ -5,6 +5,7 @@
"disk_load",
"disk_top",
"disk_filesystems",
"disk_folders",
"disk_devices",
"cpu_top"
],
+1
View File
@@ -160,6 +160,7 @@ A failure is an error naming how it failed, never an empty answer.
| `docker_disk_usage` | r | `docker system df -v`: total, active and reclaimable per kind, with the largest of each |
| `docker_networks` | r | networks, subnets, and the containers on each |
| `docker_volumes` | r | volumes, who mounts each, whether the mesh holds one of them, anonymous or not, and sizes if asked |
| `docker_volumes_remove_anonymous` | a | leftover anonymous volumes no container mounts in any state (so none the mesh holds), optionally created before a time; each other volume answered with why it stays; none can be named. **A dry run unless `dry_run` is false, which needs `why`. Never forced** (issue 1796). Sizes only when `sizes` is true (docker is slow on very many volumes); at most `limit` (default 200) per call |
| `docker_events` | r | the runtime's events over a window ending now (default 60 min, at most 24 h), without exec noise unless asked; **an exec's command line is shown with any secret it carried as `[redacted: <what it was>]`** |
| `docker_daemon_config` | r | `daemon.json` as on disk, `docker info`'s essentials, and keys the daemon has not taken yet |
| `docker_unlabelled` | r | the containers the mesh does not hold: the cleanup list |
+132 -8
View File
@@ -944,16 +944,9 @@ func (c *Client) Volumes(ctx context.Context, unmountedOnly, sizes bool) (map[st
}
size := map[string]string{}
if sizes {
out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
if err != nil {
if size, err = c.volumeSizes(ctx); err != nil {
return nil, err
}
var v struct{ Volumes []map[string]any }
if err := json.Unmarshal([]byte(out), &v); err == nil {
for _, vol := range v.Volumes {
size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
}
}
}
vols := []map[string]any{}
if len(lines(names)) > 0 {
@@ -1215,3 +1208,134 @@ func (c *Client) DaemonConfig(ctx context.Context) (map[string]any, error) {
answer["note"] = "keys read only at start take effect at the daemon's next restart; with live-restore on, a restart keeps every container running"
return answer, nil
}
// volumeSizes is every volume's size by name, from docker system df -v, which walks every volume:
// on a machine with a thousand of them it does not answer within a call's bound.
func (c *Client) volumeSizes(ctx context.Context) (map[string]string, error) {
size := map[string]string{}
out, err := c.docker(ctx, "system", "df", "--verbose", "--format", "{{json .}}")
if err != nil {
return nil, err
}
var v struct{ Volumes []map[string]any }
if err := json.Unmarshal([]byte(out), &v); err == nil {
for _, vol := range v.Volumes {
size[fmt.Sprint(vol["Name"])] = fmt.Sprint(vol["Size"])
}
}
return size, nil
}
// RemoveVolumesAsk is what docker_volumes_remove_anonymous is asked to take.
type RemoveVolumesAsk struct {
DryRun bool
Why string
CreatedBefore *time.Time
Sizes bool
Limit int // how many to remove in one call; 0 is 200
}
// RemoveAnonymousVolumes removes the leftover anonymous volumes (novox/hq issue 1796): those docker
// labels anonymous, which no container mounts in any state, so none a mesh-held container holds. The
// classification is Volumes' own, not a copy. Every other volume is answered with why it stays. Each
// is removed by its own `docker volume rm`, never forced and never `volume prune`, so one the daemon
// refuses is reported and the rest still go.
func (c *Client) RemoveAnonymousVolumes(ctx context.Context, a RemoveVolumesAsk) (map[string]any, error) {
if !a.DryRun && strings.TrimSpace(a.Why) == "" {
return nil, fmt.Errorf("why is required when dry_run is false")
}
all, err := c.Volumes(ctx, false, false)
if err != nil {
return nil, err
}
candidates := []string{}
skipped := []map[string]any{}
skip := func(name, why string) { skipped = append(skipped, map[string]any{"name": name, "why": why}) }
for _, v := range all["volumes"].([]map[string]any) {
name := fmt.Sprint(v["name"])
switch by := v["mounted_by"].([]map[string]any); {
case v["anonymous"] != true:
skip(name, "not anonymous: a named volume is data")
case v["mesh_held"] == true:
skip(name, "mounted by a container the mesh holds")
case len(by) > 0:
skip(name, fmt.Sprintf("mounted by %v (%v)", by[0]["container"], by[0]["state"]))
case a.CreatedBefore != nil && !volumeOlder(fmt.Sprint(v["created"]), *a.CreatedBefore):
skip(name, "not created before "+a.CreatedBefore.Format(time.RFC3339)+" (created "+fmt.Sprint(v["created"])+")")
default:
candidates = append(candidates, name)
}
}
answer := map[string]any{"dry_run": a.DryRun, "candidates": len(candidates)}
// Sizes are asked for, never needed: the listing walks every volume and may not answer, and
// then the call answers without them. Nothing below depends on it.
sizes := map[string]string{}
if a.Sizes {
if got, err := c.volumeSizes(ctx); err != nil {
answer["sizes_unknown"] = err.Error()
} else {
sizes = got
answer["sizes"] = sizes
}
}
sum := func(names []string) int64 {
var t int64
for _, n := range names {
t += max(Bytes(sizes[n]), 0)
}
return t
}
if a.DryRun {
answer["skipped"] = skipped
answer["would_remove"] = candidates
if len(sizes) > 0 {
answer["would_free_bytes"] = sum(candidates)
}
answer["note"] = "nothing was removed: call again with dry_run false and why to remove them (limit volumes at most per call)"
return answer, nil
}
limit := a.Limit
if limit <= 0 {
limit = 200
}
take := candidates
if len(take) > limit {
take = candidates[:limit]
}
removed := []string{}
// Chunks of a hundred in one `docker volume rm`, so a thousand volumes are ten calls; a chunk
// docker refuses any of is done again one by one, so what it refuses is named.
for i := 0; i < len(take); i += 100 {
chunk := take[i:min(i+100, len(take))]
if _, err := c.docker(ctx, append([]string{"volume", "rm"}, chunk...)...); err == nil {
removed = append(removed, chunk...)
continue
}
for _, name := range chunk {
if _, err := c.docker(ctx, "volume", "rm", name); err != nil {
// The batch took it before it refused another (or before its bound ran out):
// a volume that is no longer there is a removed one.
if strings.Contains(strings.ToLower(err.Error()), "no such volume") {
removed = append(removed, name)
continue
}
skip(name, "docker refused: "+err.Error())
continue
}
removed = append(removed, name)
}
}
answer["skipped"] = skipped
answer["removed"], answer["count"], answer["why"] = removed, len(removed), a.Why
answer["remaining"] = len(candidates) - len(take)
if len(sizes) > 0 {
answer["freed_bytes"] = sum(removed)
}
return answer, nil
}
// volumeOlder is whether a volume's CreatedAt is before the cutoff; one that cannot be read is not.
func volumeOlder(created string, cutoff time.Time) bool {
t, err := time.Parse(time.RFC3339Nano, created)
return err == nil && t.Before(cutoff)
}
@@ -431,3 +431,145 @@ func TestEventsCoveringTheWholeWindowSayItIsCompleteAndFilterTheTypeHere(t *test
t.Fatalf("a full buffer reaching past the window: %v", got)
}
}
// 1796/AC1: only anonymous volumes nothing mounts go, each answered; a named, a mounted and a too young one stay with why.
func TestRemoveAnonymousVolumesTakesOnlyTheUnmountedAnonymousOnes(t *testing.T) {
f := machine().
on("docker volume ls --quiet", Ran{Stdout: "webdata\nanonmounted\nold\nyoung\nnamed\n"}).
on("docker volume inspect", Ran{Stdout: `[
{"Name":"webdata","Driver":"local","CreatedAt":"2026-01-01T00:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
{"Name":"old","Driver":"local","CreatedAt":"2026-09-01T00:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
{"Name":"young","Driver":"local","CreatedAt":"2026-10-04T11:00:00Z","Labels":{"com.docker.volume.anonymous":""}},
{"Name":"named","Driver":"local","CreatedAt":"2026-01-01T00:00:00Z"}]`}).
on("docker system df --verbose", Ran{Stdout: `{"Volumes":[{"Name":"old","Size":"1.5MB"},{"Name":"young","Size":"1kB"}]}`}).
on("docker volume rm old", Ran{Stdout: "old\n"})
c := client(f, 1000)
before := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
got, err := c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true, Sizes: true, CreatedBefore: &before})
if err != nil {
t.Fatal(err)
}
if f.ran("docker volume rm") || !reflect.DeepEqual(got["would_remove"], []string{"old"}) {
t.Fatalf("dry run: %v %+v", got, f.calls)
}
if got["would_free_bytes"] != int64(1500000) || got["sizes"].(map[string]string)["old"] != "1.5MB" {
t.Fatalf("dry run size: %v", got)
}
skipped := got["skipped"].([]map[string]any)
why := map[string]string{}
for _, s := range skipped {
why[s["name"].(string)] = s["why"].(string)
}
if len(why) != 3 || !strings.Contains(why["webdata"], "mesh holds") || why["young"] == "" || why["named"] == "" {
t.Fatalf("skipped: %v", skipped)
}
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "leftovers", Sizes: true, CreatedBefore: &before})
if err != nil {
t.Fatal(err)
}
if !f.ran("docker volume rm old") || got["count"] != 1 || !reflect.DeepEqual(got["removed"], []string{"old"}) {
t.Fatalf("removed: %v %+v", got, f.calls)
}
if got["freed_bytes"] != int64(1500000) {
t.Fatalf("freed: %v", got)
}
for _, c := range f.calls {
if len(c.args) > 1 && c.args[0] == "volume" && c.args[1] == "rm" && (len(c.args) != 3 || c.args[2] != "old") {
t.Errorf("reached too far: %v", c.args)
}
}
}
// 1796/AC2: a volume that is mounted by a mesh-held container is never removed; the tool takes no name.
func TestRemoveAnonymousVolumesRefusesANamedVolumeAndKeepsWhatTheMeshHolds(t *testing.T) {
for _, tool := range tools(client(&fake{}, 1000)) {
if tool.Name != "docker_volumes_remove_anonymous" {
continue
}
for _, k := range []string{"name", "names", "volume", "volumes"} {
if _, err := tool.Run(map[string]any{k: "x", "dry_run": false, "why": "w"}); err == nil {
t.Errorf("%s accepted", k)
}
}
if _, err := tool.Run(map[string]any{"dry_run": false}); err == nil {
t.Error("a removal without why was accepted")
}
if _, err := tool.Run(map[string]any{"created_before": "yesterday"}); err == nil {
t.Error("a bad time was accepted")
}
return
}
t.Fatal("tool not served")
}
// 1796/AC3: on a machine with a very large set of volumes docker's size listing does not answer in
// time; the tool still answers, and removal never depends on it.
func TestRemoveAnonymousVolumesAnswersWhenTheSizeListingDoesNot(t *testing.T) {
f := (&fake{}).
on("docker ps --all --quiet --no-trunc", Ran{}).
on("docker volume ls --quiet", Ran{Stdout: "a1\na2\na3\n"}).
on("docker volume inspect", Ran{Stdout: `[{"Name":"a1","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a2","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a3","Labels":{"com.docker.volume.anonymous":""}}]`}).
on("docker system df", Ran{Status: 124, Err: "no answer within 20 s"}).
on("docker volume rm", Ran{Stdout: "ok\n"})
c := client(f, 1000)
got, err := c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true})
if err != nil || f.ran("docker system df") || !reflect.DeepEqual(got["would_remove"], []string{"a1", "a2", "a3"}) {
t.Fatalf("default dry run: %v %v %+v", got, err, f.calls)
}
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{DryRun: true, Sizes: true})
if err != nil || !reflect.DeepEqual(got["would_remove"], []string{"a1", "a2", "a3"}) || got["sizes_unknown"] == nil {
t.Fatalf("sizes asked, df silent: %v %v", got, err)
}
got, err = c.RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w", Sizes: true, Limit: 2})
if err != nil || got["count"] != 2 || got["remaining"] != 1 || !f.ran("docker volume rm a1 a2") {
t.Fatalf("removal: %v %v %+v", got, err, f.calls)
}
}
// 1796/AC4: a batch docker half-did is settled by name: what is already gone counts as removed, what is in use stays with why.
func TestRemoveAnonymousVolumesCountsWhatAFailedBatchAlreadyRemoved(t *testing.T) {
f := (&fake{}).
on("docker ps --all --quiet --no-trunc", Ran{}).
on("docker volume ls --quiet", Ran{Stdout: "a1\na2\na3\n"}).
on("docker volume inspect", Ran{Stdout: `[{"Name":"a1","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a2","Labels":{"com.docker.volume.anonymous":""}},{"Name":"a3","Labels":{"com.docker.volume.anonymous":""}}]`}).
on("docker volume rm a1 a2 a3", Ran{Status: 1, Stdout: "a1\n", Stderr: "Error response from daemon: remove a2: volume is in use - [c1]\n"}).
on("docker volume rm a1", Ran{Status: 1, Stderr: "Error response from daemon: get a1: no such volume\n"}).
on("docker volume rm a2", Ran{Status: 1, Stderr: "Error response from daemon: remove a2: volume is in use - [c1]\n"}).
on("docker volume rm a3", Ran{Stdout: "a3\n"})
got, err := client(f, 1000).RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w"})
if err != nil {
t.Fatal(err)
}
sk := got["skipped"].([]map[string]any)
if !reflect.DeepEqual(got["removed"], []string{"a1", "a3"}) || got["count"] != 2 || len(sk) != 1 || sk[0]["name"] != "a2" || !strings.Contains(sk[0]["why"].(string), "in use") {
t.Fatalf("%v", got)
}
}
// 1796/AC5: removal goes in batches of a hundred names.
func TestRemoveAnonymousVolumesRemovesInBatchesOfAHundred(t *testing.T) {
names, entries := []string{}, []string{}
for i := 0; i < 250; i++ {
n := fmt.Sprintf("v%03d", i)
names = append(names, n)
entries = append(entries, `{"Name":"`+n+`","Labels":{"com.docker.volume.anonymous":""}}`)
}
f := (&fake{}).
on("docker ps --all --quiet --no-trunc", Ran{}).
on("docker volume ls --quiet", Ran{Stdout: strings.Join(names, "\n")}).
on("docker volume inspect", Ran{Stdout: "[" + strings.Join(entries, ",") + "]"}).
on("docker volume rm", Ran{Stdout: "ok\n"})
got, err := client(f, 1000).RemoveAnonymousVolumes(context.Background(), RemoveVolumesAsk{Why: "w", Limit: 250})
if err != nil || got["count"] != 250 {
t.Fatalf("%v %v", got, err)
}
sizes := []int{}
for _, c := range f.calls {
if len(c.args) > 1 && c.args[0] == "volume" && c.args[1] == "rm" {
sizes = append(sizes, len(c.args)-2)
}
}
if !reflect.DeepEqual(sizes, []int{100, 100, 50}) {
t.Fatalf("batches %v", sizes)
}
}
+35
View File
@@ -11,6 +11,7 @@ import (
"math"
"os"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -286,6 +287,40 @@ func tools(c *Client) []stdio.Tool {
return c.Volumes(ctx, flag(args, "unmounted", false), flag(args, "sizes", false))
},
},
{
Name: "docker_volumes_remove_anonymous",
Description: "Remove the leftover anonymous volumes (novox/hq issue 1796): only a volume docker labels anonymous (as docker_volumes says), which no container mounts in any state " +
"and so none the mesh holds, and with created_before only one created before it. Every other volume is answered with why it stays; a named volume is data and is never taken, " +
"and no volume can be named. Each is removed on its own, never forced; the answer has the count and names removed. With sizes true the answer has each volume's size and the bytes freed; docker's size listing is slow on many volumes, so it is asked for, and when it does not answer the call answers without it. At most limit are removed per call, in batches. " +
"A dry run by default; dry_run false needs why. Replaces docker volume rm and docker volume prune. (a)",
Input: map[string]any{
"dry_run": map[string]any{"type": "boolean", "description": "list only (default true)"},
"why": map[string]any{"type": "string", "description": "why: required when dry_run is false"},
"created_before": map[string]any{"type": "string", "description": "RFC 3339 time: only volumes created before it (default: any age)"},
"sizes": map[string]any{"type": "boolean", "description": "also read each volume's size and the bytes freed (default false: docker walks every volume for it and on a machine with very many may not answer; the call then answers without sizes)"},
"limit": map[string]any{"type": "integer", "description": "how many to remove in one call (default 200, at most 2000); the answer says how many remain, call again for them"},
},
Run: func(args map[string]any) (any, error) {
for _, k := range []string{"name", "names", "volume", "volumes"} {
if _, ok := args[k]; ok {
return nil, fmt.Errorf("%s: this tool takes no volume by name; it removes only unmounted anonymous volumes", k)
}
}
limit, err := bounded(args, "limit", 200, 2000)
if err != nil {
return nil, err
}
ask := RemoveVolumesAsk{DryRun: flag(args, "dry_run", true), Why: optional(args, "why"), Sizes: flag(args, "sizes", false), Limit: limit}
if s := optional(args, "created_before"); s != "" {
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return nil, fmt.Errorf("created_before must be an RFC 3339 time such as 2026-10-01T00:00:00Z")
}
ask.CreatedBefore = &t
}
return c.RemoveAnonymousVolumes(ctx, ask)
},
},
{
Name: "docker_events",
Description: "What the runtime did in a window ending now (default the last 60 minutes, at most 24 hours): containers created, started, died, health changes, images pulled — with mesh_held. Exec events are left out unless asked; " +
+1
View File
@@ -33,6 +33,7 @@
"docker_disk_usage",
"docker_networks",
"docker_volumes",
"docker_volumes_remove_anonymous",
"docker_events",
"docker_daemon_config",
"docker_unlabelled",
+44
View File
@@ -0,0 +1,44 @@
// **A consumer that cannot be bound yet is bound again, never fatal** (novox/hq issue 1793). When the tool runtime
// starts this bundle, the runtime's answer to "bind this module's consumer" can be "no responders" for a few seconds (the
// runtime restarted by a delivery, the bus not yet serving the stream). That rejection, left unhandled at the top of
// index.ts, ended the bundle: nothing heard `mesh-controller.checked` again, and a verdict that landed later never
// reached the forge's statuses. A bind that fails is said and asked again with a growing wait until it holds.
/** How long to wait before the nth new attempt (1-based): 2 s, 5 s, 10 s, then every 30 s. */
export function bindWait(attempt: number): number {
return [2_000, 5_000, 10_000][attempt - 1] ?? 30_000;
}
/**
* Binds with `bind`, asking again after each failure, until it holds. `sleep` is the wait; `say` how a failure is said.
* Never rejects; resolves once the bind holds.
*/
export async function bindUntilHeld(what: string, bind: () => Promise<unknown>,
sleep: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms)),
say: (line: string) => void = (line) => console.error(line)): Promise<void> {
for (let attempt = 1; ; attempt++) {
try {
await bind();
if (attempt > 1) say(`[gitea] ${what} is bound after ${attempt} attempts`);
return;
} catch (err) {
const wait = bindWait(attempt);
say(`[gitea] ${what} could not be bound (${err instanceof Error ? err.message : String(err)}); asking again in ${wait / 1000}s`);
await sleep(wait);
}
}
}
/**
* **Starts the binds without making the bundle wait for them** (novox/hq issue 1793). The launcher imports the bundle
* and then answers the runtime's handshake, which the runtime gives 10 s and then kills the bundle without starting it
* again; a bind awaited at the top of the module that stays refused longer ends the bundle all the same. So this
* returns at once: the binds run in the order given, each until it holds, and `done` is called after the last.
*/
export function bindInBackground(binds: [what: string, bind: () => Promise<unknown>][], done: () => void,
sleep?: (ms: number) => Promise<void>, say?: (line: string) => void): void {
void (async () => {
for (const [what, bind] of binds) await bindUntilHeld(what, bind, sleep, say);
done();
})();
}
+15 -11
View File
@@ -47,6 +47,7 @@ import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, commentFor, composedGate, groupGateS
import { GateWriter, clonedAs, type BuildStarted } from "./gate.js";
import { HOOK_PATH, HOOK_PORT, Pace, claimHead, ensureSystemHook, eventsTool, keptSecret, receiver, type ForgeEvent } from "./webhook.js";
import { keepSiblings, noteComposed, noteVerdict, readSiblings, siblingsFile, siblingsOf, weighSiblings } from "./siblings.js";
import { bindInBackground } from "./bind.js";
import { announcementOf, keepTook, lateVerdictComment, outOfDateWords, sameCommit, tookFile, weighTrunks } from "./trunk.js";
// Without a way to a token — configured, or mintable with the admin account (token.ts) — there is
@@ -590,15 +591,18 @@ if (gitea) {
tick(() => pollRepos(client), 60_000);
tick(() => pollMerged(client), () => pace.every());
tick(() => pollPulls(client), () => pace.every());
await on("mesh-controller.checked", (event) => {
const c = (event.body ?? {}) as Checked;
// A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts).
// A verdict of a check asked before the one now pending is an older run's, and dropped (novox/hq issue 456).
if (!c.group && c.owner && c.repo && c.commit) {
return gates.verdict(c.owner, c.repo, c.commit, () => setVerdict(client, event), c.id);
}
return setVerdict(client, event);
});
await on("node-build-agent.started", buildStarted);
console.log("[gitea] watching for new repositories and merged pull requests");
// Bound again until they hold, and not awaited (bind.ts, issue 1793): the runtime kills a bundle that has not answered its
// handshake in 10 s, so the import must end while a refused bind is still asked again.
bindInBackground([
["mesh-controller.checked", () => on("mesh-controller.checked", (event) => {
const c = (event.body ?? {}) as Checked;
// A head's own verdict takes its turn after every write of that head's gate already asked for (gate.ts).
// A verdict of a check asked before the one now pending is an older run's, and dropped (novox/hq issue 456).
if (!c.group && c.owner && c.repo && c.commit) {
return gates.verdict(c.owner, c.repo, c.commit, () => setVerdict(client, event), c.id);
}
return setVerdict(client, event);
})],
["node-build-agent.started", () => on("node-build-agent.started", buildStarted)],
], () => console.log("[gitea] watching for new repositories and merged pull requests"));
}
+9 -1
View File
@@ -12,7 +12,7 @@ import { mergeableUnder, newestByContext } from "./checks.js";
import { MERGE_METHODS, Refused, checkName } from "./guard.js";
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
import { approvedHeadsOf, groupRefusal, memberKey, namesSiblings, notApproved, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js";
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk } from "./trunk.js";
import { announcementOf, askedWords, outOfDateWords, sameCommit, tookFile, tookWords, twelve, verdictAtTrunk, waitsForComposed } from "./trunk.js";
/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */
const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/;
@@ -117,6 +117,14 @@ export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition
: at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against";
const took = tookWords(tookFile(deps.stateDir), `${owner}/${repo}`);
const head = twelve(pull.head_sha), now = twelve(trunkHead);
// **A member that waits for its group's composed check is not announced again** (novox/hq issue 1106): its own
// check can only say "composed" once more, so an ask here brought a verdict naming no trunk and the next merge was
// refused the same way for ever. The composed check is mesh-delivery's to ask; this says whose verdict is awaited.
if (waitsForComposed(checks[CHECK_CONTEXT])) {
throw new Refused(`${owner}/${repo}#${number} is not merged: its merge gate ${checks[CHECK_CONTEXT]?.description}; the ` +
`verdict that names its trunk commit is its group's composed check's, and mesh-delivery asks it, this tool asks no check ` +
`(main is now ${now}). Ask again once the group's composed check has passed. Nothing was merged`);
}
if (at.onItsWay) {
throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` +
`${now}. Its check is already asked and not yet answered (${checks[CHECK_CONTEXT]?.description}), so it is not ` +
+5 -1
View File
@@ -226,6 +226,10 @@ function repoWrong(repo: Layer | undefined, facts: RepoCheckFacts): boolean {
return !!repo && repoStatus(repo, facts).state !== "success";
}
/** What a group member's gate says while the group's composed check is the one that judges it: the member's own check can say
* nothing more of it, so asking that check again brings no verdict that names a trunk (novox/hq issue 1106). */
export const WAITS_FOR_COMPOSED = "waits for its group's composed check";
/** The forge's status for the gate. */
export function statusFor(c: Checked): CommitStatus {
const gate = c.gate ?? { verdict: c.verdict, summary: c.summary };
@@ -260,7 +264,7 @@ export function statusesFor(c: Checked, facts: RepoCheckFacts = {}, gateNow?: st
// set. The gate stays pending, saying it waits for the group, unless a composed verdict already stands on it.
const out: CommitStatus[] = [];
if (!(gateNow ?? "").includes(" composed: ")) {
out.push({ state: "pending", context: CHECK_CONTEXT, description: "waits for its group's composed check" });
out.push({ state: "pending", context: CHECK_CONTEXT, description: WAITS_FOR_COMPOSED });
}
const repo = c["repo-check"];
if (repo) out.push(judgedFirst(c, repoStatus(repo, facts)));
+58
View File
@@ -0,0 +1,58 @@
// **A consumer that cannot be bound yet is bound again** (novox/hq issue 1793): the runtime's "no responders" at the
// bundle's start, seen 2026-10-22 00:28:37 on mesh-catalog#427's verdict, ended the bundle and the verdict was never
// written on the forge.
import { test } from "node:test";
import assert from "node:assert/strict";
import { bindInBackground, bindUntilHeld, bindWait } from "../dist/bind.js";
test("a bind refused with no responders is asked again until it holds, and says so", async () => {
let calls = 0;
const waits: number[] = [];
const said: string[] = [];
await bindUntilHeld("mesh-controller.checked", async () => {
calls++;
if (calls < 4) throw new Error("binding gitea's consumer novox_gitea: nats: no responders available for request");
}, async (ms) => { waits.push(ms); }, (l) => said.push(l));
assert.equal(calls, 4);
assert.deepEqual(waits, [2_000, 5_000, 10_000]);
assert.equal(said.length, 4);
assert.match(said[0], /no responders available for request.*asking again in 2s/);
assert.match(said[3], /is bound after 4 attempts/);
});
test("a bind that holds the first time says nothing and waits for nothing", async () => {
const said: string[] = [];
await bindUntilHeld("x", async () => {}, async () => assert.fail("waited"), (l) => said.push(l));
assert.deepEqual(said, []);
});
test("the wait grows to 30 s and stays", () => {
assert.deepEqual([1, 2, 3, 4, 50].map(bindWait), [2_000, 5_000, 10_000, 30_000, 30_000]);
});
test("the binds start without being waited for: the caller returns while a bind keeps refusing, and they go in order", async () => {
const order: string[] = [];
let finished = false;
let refusals = 0;
const gate = new Promise<void>((r) => setTimeout(r, 0));
bindInBackground([
["first", async () => { refusals++; if (refusals < 3) throw new Error("nats: no responders available for request"); order.push("first"); }],
["second", async () => { order.push("second"); }],
], () => { finished = true; }, async () => { await gate; }, () => {});
// Returned at once, with the first bind refused and nothing done.
assert.equal(finished, false);
assert.deepEqual(order, []);
await new Promise((r) => setTimeout(r, 20));
assert.deepEqual(order, ["first", "second"]);
assert.equal(finished, true);
});
test("a bind that never holds does not stop the caller from returning", () => {
let returned = false;
bindInBackground([["x", async () => { throw new Error("no responders"); }]], () => assert.fail("done"),
() => new Promise(() => {}), () => {});
returned = true;
assert.equal(returned, true);
});
+38
View File
@@ -0,0 +1,38 @@
// **The bundle's import ends while a bind is refused** (novox/hq issue 1793). The launcher imports index.js and then answers
// node-tools' handshake, which has 10 s before the bundle is killed and never started again. A bind awaited at the top
// of index.ts, refused for longer, ended the bundle at 00:28:37 on 2026-10-22. Run in a child: the polls' timers keep a
// process alive, and the child says "imported" and exits as soon as the import resolves.
import { test } from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { dirname, join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const here = dirname(fileURLToPath(import.meta.url));
test("importing the bundle resolves while the broker keeps refusing the binds", async () => {
const index = pathToFileURL(join(here, "..", "dist", "index.js")).href;
const script = `
const { useBroker } = await import("@novox/mesh-sdk/messaging");
useBroker(() => ({
subscribe: async () => { throw new Error("binding gitea's consumer novox_gitea: nats: no responders available for request"); },
publish: async () => {}, request: async () => { throw new Error("no responders"); },
handle: async () => () => {}, close: async () => {},
}));
await import(${JSON.stringify(index)});
console.log("imported");
process.exit(0);`;
const env = { ...process.env, MESH_GITEA_TOKEN: "x", MESH_GITEA_URL: "http://127.0.0.1:1" };
for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE"]) delete env[k];
const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] });
let out = "";
let err = "";
child.stdout.on("data", (d) => (out += d));
child.stderr.on("data", (d) => (err += d));
const ended = new Promise<void>((r) => child.on("exit", () => r()));
const timer = setTimeout(() => child.kill("SIGKILL"), 3000);
await ended;
clearTimeout(timer);
assert.match(out, /imported/, `the import did not resolve within 3 s while a bind was refused; stderr: ${err.slice(0, 600)}`);
});
+24
View File
@@ -339,6 +339,30 @@ test("gitea_merge_pull_request refuses a verdict judged against an earlier trunk
assert.equal(state.emitted.length, 4, "a merge asks no check");
});
// **A group member whose gate waits for its group's composed check is not announced again by the merge tool** (novox/hq issue
// 1106): the member's own check can only say "composed" again, so each refusal's ask brought a verdict that names no trunk and
// the next merge was refused the same way, for ever. The tool says whose verdict is awaited and asks nothing.
test("gitea_merge_pull_request asks no check of a member that waits for its group's composed check", async () => {
const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED });
state.trunk = TRUNK;
const before = state.emitted.length;
state.statuses = [{ context: "mesh/merge-gate", state: "pending", description: "waits for its group's composed check" },
{ context: "mesh/repo-check", state: "success", description: "pass: y" }];
for (let i = 0; i < 3; i++) {
await assert.rejects(merge(), (err: Error) => {
assert.match(err.message, /waits for its group's composed check/);
assert.match(err.message, /mesh-delivery asks it, this tool asks no check/);
assert.match(err.message, /Nothing was merged/);
return true;
});
}
assert.equal(state.emitted.length, before, "no check is announced again and again");
assert.equal(state.merged, false);
// Once the composed verdict names the member's unmoved trunk, it merges.
state.statuses = judgedAt(TRUNK);
assert.equal(((await merge()) as Record<string, unknown>).merged, true);
});
// A merge lands only the head that was reviewed (novox/hq issue 381): mesh-controller#197 was reviewed at one head,
// force-pushed, checked green at the new one, and merged there, because the tool merged whatever the head was.
test("gitea_merge_pull_request merges only the head it is named, and refuses one that is not the head", async () => {
+6 -1
View File
@@ -17,7 +17,7 @@
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import type { CommitStatus, GiteaClient, GiteaPull, HeadCheck } from "./client.js";
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT } from "./pulls.js";
import { CHECK_CONTEXT, REPO_CHECK_CONTEXT, WAITS_FOR_COMPOSED } from "./pulls.js";
/** A commit as a verdict names it: its first twelve characters. */
export function twelve(sha: string | undefined): string {
@@ -58,6 +58,11 @@ export function checkOnItsWay(gate: HeadCheck | undefined): boolean {
return d.startsWith("waiting in the build queue") || d.startsWith("building on ") || d.startsWith("checking again");
}
/** Whether a gate says it waits for its group's composed check. */
export function waitsForComposed(gate: HeadCheck | undefined): boolean {
return !!gate && gate.state === "pending" && (gate.description ?? "").startsWith(WAITS_FOR_COMPOSED);
}
/**
* The statuses a trunk's move puts back to pending on one open pull request's head, from what the forge holds on it
* now: none when it has no merge check yet, when a check is pending (on its way, or already out of date — its
+13 -1
View File
@@ -16,6 +16,18 @@ installation's are settings on the assignment, `settings set mailu <file>`:
header from — the address the proxy forwards with. The front's own hostname is the name of its
`web` route, told to it by the mesh.
## Mail client autoconfiguration
`mailu-automx` answers what mail clients ask for their settings (Thunderbird, Outlook, Apple) for the domains in
`AMX_MAIL_DOMAINS`, naming the IMAP and SMTP servers and ports of mailu's environment (`AMX_IMAP_ADDRESS`,
`AMX_IMAP_PORT`, `AMX_SMTP_ADDRESS`, `AMX_SMTP_PORT`). It is `cmd/automx`, a Go program of the standard library
only in this module's `go.mod` (ADR 0480 clauses 1, 2, 9). It runs in the mirrored alpine image the route proxy's
`trust` step runs, from the `program-automx` archive mounted read-only at `/opt/mesh`; the container is restarted
with the archive and with `mailu.env`, which the program reads at each start. It keeps no database. The file the
Python automx kept in `data-automx` is left where it is on the machine, not deleted by the mesh (ADR 0480
clause 10). The answers it gave before are the fixtures in `cmd/automx/testdata/captured`, with how they were
taken (`CAPTURED`) and what the program answers differently (`DIFFERENCES`).
## Files it does not know
Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416).
@@ -23,7 +35,7 @@ Each directory or archive this module declares says how it treats a file it did
- `${dir:mesh-state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:grants}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-automx}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- the archive `program-automx` (`${dir:state}/automx`): everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-certs}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-data}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
- `${dir:data-dkim}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2).
-87
View File
@@ -1,87 +0,0 @@
// What the autoconfiguration module (automx2) holds, as stored: its provider, server, domain and server_domain
// rows, read from its SQLite file inside its container with the sqlite3 program the image carries, opened
// read-only. Nothing is written and nothing is interpreted: the rows are the evidence for what automx answers
// to mail clients (novox/hq issue 1630, ADR 0480), and the reason the answer may be empty.
import type { Exec } from "./health.js";
export const AUTOMX_DATABASE = "/data/db.sqlite";
/** The tables read, each with its columns and its ordering. Fixed text: the tool takes no query from its caller. */
export const AUTOMX_TABLES: Record<string, { columns: string[]; order: string }> = {
provider: { columns: ["id", "name", "short_name"], order: "id" },
server: { columns: ["id", "prio", "name", "port", "type", "socket_type", "user_name", "authentication"], order: "id" },
domain: { columns: ["id", "name", "provider_id", "ldapserver_id"], order: "id" },
server_domain: { columns: ["server_id", "domain_id"], order: "domain_id, server_id" },
};
/**
* The one statement that reads a table. The sqlite3 in the mailu-automx image does not know `-json` (it answers
* "sqlite3: Error: unknown option: -json", issue 1630), so each column is printed as `<type>:<hex of its bytes>`, which
* the oldest sqlite3 prints in its default list mode: no quoting, no separator or newline inside a cell, and a NULL
* (`null:`) apart from an empty text (`text:`).
*/
export function selectOf(table: string): string {
const t = AUTOMX_TABLES[table];
return `SELECT ${t.columns.map((c) => `typeof(${c}) || ':' || hex(${c})`).join(", ")} FROM ${table} ORDER BY ${t.order}`;
}
export type Row = Record<string, string | number | null>;
export type AutomxRows = {
database: string;
tables: Record<string, Row[]>;
/** servers whose name is empty: the client is told an empty host name for them */
empty_names: { id: number; type: string }[];
/** tables that could not be read, each with what sqlite3 or docker said on one line */
unreadable: Record<string, string>;
};
/** One cell as selectOf prints it: `<type>:<hex>`; text is the bytes read as UTF-8, a number is read from its text. */
function cellOf(cell: string): string | number | null {
const at = cell.indexOf(":");
if (at < 0 || cell.length - at - 1 > 0 && !/^([0-9A-Fa-f]{2})+$/.test(cell.slice(at + 1))) throw new Error(`sqlite3 printed ${JSON.stringify(cell.slice(0, 40))}, not a cell`);
const type = cell.slice(0, at);
const bytes = Buffer.from(cell.slice(at + 1), "hex").toString("utf8");
switch (type) {
case "null": return null;
case "integer": case "real": return Number(bytes);
default: return bytes;
}
}
/** sqlite3 prints one line per row, cells apart by `|`, and nothing at all for a table without rows. */
export function parseRows(table: string, output: string): Row[] {
const columns = AUTOMX_TABLES[table].columns;
const rows: Row[] = [];
for (const line of output.split("\n")) {
if (line.trim() === "") continue;
const cells = line.replace(/\r$/, "").split("|");
if (cells.length !== columns.length) throw new Error(`sqlite3 printed ${cells.length} cells for ${columns.length} columns`);
rows.push(Object.fromEntries(columns.map((c, i) => [c, cellOf(cells[i])])));
}
return rows;
}
/** The command that reads one table: read-only, one fixed statement. */
export function readCommand(table: string): string[] {
return ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf(table)];
}
/** Read every table; one that fails is said in `unreadable` and does not hide the others. */
export async function readAutomx(exec: Exec, container: string): Promise<AutomxRows> {
const answer: AutomxRows = { database: AUTOMX_DATABASE, tables: {}, empty_names: [], unreadable: {} };
for (const table of Object.keys(AUTOMX_TABLES)) {
try {
answer.tables[table] = parseRows(table, await exec(container, readCommand(table)));
} catch (e) {
const err = e as { stderr?: string; message?: string };
const text = (err?.stderr ?? "").trim() || (err?.message ?? String(e)).trim();
answer.unreadable[table] = text.split("\n").map((l) => l.trim()).filter(Boolean)[0] ?? "it failed and said nothing";
}
}
for (const s of answer.tables.server ?? []) {
if (String(s.name ?? "").trim() === "") answer.empty_names.push({ id: Number(s.id), type: String(s.type) });
}
return answer;
}
-44
View File
@@ -1,44 +0,0 @@
# automx2 — the autoconfig/autodiscover responder, carried by the mailu module as its own
# artifact: it is a config-baked sidecar of this mail server, not a standalone application
# (novox/hq ADR 0015 draws that line at applications).
#
# The base is named rather than pinned (novox/hq issue 044): declared in module.json's
# `build.on`. The build context is the module's own directory; every ADD says so.
ARG PYTHON_BASE
FROM ${PYTHON_BASE}
RUN apk add --no-cache bash sqlite
WORKDIR /automx2
ADD automx/files/setupvenv.sh /automx2/setupvenv.sh
ADD automx/files/start /automx2/start
ADD automx/files/setup /automx2/setup
ADD automx/files/setup-db /automx2/setup-db
ADD automx/files/add-domains /automx2/add-domains
RUN chmod u+x setupvenv.sh start add-domains setup setup-db
RUN ./setupvenv.sh \
&& . .venv/bin/activate \
&& pip install automx2==2021.6
# The launcher `start` expects. In the predecessor's image this wrapper appeared during a build
# step that never made it into the files this module carries — the image worked and the recipe
# could not reproduce it. Written here explicitly, verbatim from the proven image, so the build
# is the whole truth about the image again.
RUN mkdir -p .venv/scripts && printf '%s\n' \
'#!/usr/bin/env bash' \
'set -euo pipefail' \
'. .venv/bin/activate' \
"export FLASK_ENV='production'" \
"export FLASK_APP='automx2.server:app'" \
'flask "$@"' > .venv/scripts/flask.sh && chmod +x .venv/scripts/flask.sh
ENV AUTOMX2_CONF=/etc/automx2.conf
ADD automx/files/automx2.conf /etc/automx2.conf
# VOLUME deliberately absent: the anonymous /data volume is exactly what lost db.sqlite on
# every recreate (measured on novox 2026-08-10). The manifest binds a real directory instead.
ENTRYPOINT ["/bin/sh"]
CMD ["./start"]
EXPOSE 4243
-49
View File
@@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -e
echo "${MAIL_DOMAINS}"
# Split domains into array
IFS=', ' read -r -a array <<< "${AMX_MAIL_DOMAINS}"
# User configurable section -- START
PROVIDER_ID=001
SQL_CMD="";
# Iterate domains resulting from split on second arg
for element in "${array[@]}"
do
# Set vars
DOMAIN=$element
PROVIDER_NAME=$DOMAIN
PROVIDER_SHORTNAME=$DOMAIN
# Optional LDAP server
#LDAP_SERVER="ldap.${DOMAIN}"
# User configurable section -- END
s1_id=$((PROVIDER_ID + 1))
s2_id=$((PROVIDER_ID + 2))
s3_id=$((PROVIDER_ID + 3))
dom_id=$((PROVIDER_ID + 4))
s3_id='NULL'
SQL_CMD=$(cat <<EOT
$SQL_CMD
INSERT INTO provider(id, name, short_name) VALUES(${PROVIDER_ID}, '${PROVIDER_NAME}', '${PROVIDER_SHORTNAME}');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES(${s1_id}, ${AMX_IMAP_PORT}, 'imap', '${AMX_IMAP_SERVER}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES(${s2_id}, ${AMX_SMTP_PORT}, 'smtp', '${AMX_SMTP_ADDRESS}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO domain(id, name, provider_id, ldapserver_id) VALUES(${dom_id}, '${DOMAIN}', ${PROVIDER_ID}, ${s3_id});
INSERT INTO server_domain(server_id, domain_id) VALUES(${s1_id}, ${dom_id});
INSERT INTO server_domain(server_id, domain_id) VALUES(${s2_id}, ${dom_id});
EOT
)
PROVIDER_ID=$((PROVIDER_ID+10))
done
echo -e ${SQL_CMD}
echo -e ${SQL_CMD} | sqlite3 /data/db.sqlite
-21
View File
@@ -1,21 +0,0 @@
[automx2]
# A typical production setup would use loglevel = WARNING
loglevel = WARNING
# Echo SQL commands into log? Used for debugging.
db_echo = false
# In-memory SQLite database
# db_uri = sqlite:///:memory:
# SQLite database in a UNIX-like file system
db_uri = sqlite:////data/db.sqlite
# MySQL database on a remote server. This example does not use an encrypted
# connection and is therefore *not* recommended for production use.
#db_uri = mysql://username:password@server.example.com/db
# Number of proxy servers between automx2 and the client (default: 0).
# If your logs only show 127.0.0.1 or ::1 as the source IP for incoming
# connections, proxy_count probably needs to be changed.
proxy_count = 1
-12
View File
@@ -1,12 +0,0 @@
#!/usr/bin/env bash
set -e
if [ ! -e /data/db.sqlite ]; then
# DB SETUP
echo "SETTING UP DB"
./setup-db
echo "ADDING DOMAINS"
# Add the domains
./add-domains
fi
-84
View File
@@ -1,84 +0,0 @@
#!/usr/bin/env bash
set -e
# LDAP-Server
LDAP=$(cat <<EOT
CREATE TABLE ldapserver(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
port INT NOT NULL,
use_ssl INT NOT NULL,
search_base TEXT NOT NULL,
search_filter TEXT NOT NULL,
attr_uid TEXT NOT NULL,
attr_cn TEXT NOT NULL,
bind_password TEXT NOT NULL,
bind_user TEXT NOT NULL
);
EOT
)
# Provider
PROVIDER=$(cat <<EOT
CREATE TABLE provider(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
short_name TEXT NOT NULL
);
EOT
)
# Server
SERVER=$(cat <<EOT
CREATE TABLE server(
id INT PRIMARY KEY NOT NULL,
prio INT NOT NULL DEFAULT 10,
name TEXT NOT NULL,
port INT NOT NULL,
type TEXT NOT NULL,
socket_type TEXT NOT NULL,
user_name TEXT NOT NULL,
authentication TEXT NOT NULL
);
EOT
)
# Domain
DOMAIN=$(cat <<EOT
CREATE TABLE domain(
id INT PRIMARY KEY NOT NULL,
name TEXT NOT NULL,
provider_id INT NOT NULL,
ldapserver_id INT NULL,
FOREIGN KEY(ldapserver_id) REFERENCES ldapserver(id),
FOREIGN KEY(provider_id) REFERENCES provider(id)
);
CREATE UNIQUE INDEX domain_name ON domain(name);
EOT
)
# Server-Domain
SERVER_DOMAIN=$(cat <<EOT
CREATE TABLE server_domain(
server_id INT NOT NULL,
domain_id INT NOT NULL,
FOREIGN KEY(server_id) REFERENCES server(id),
FOREIGN KEY(domain_id) REFERENCES domain(id)
);
EOT
)
## TODO Foreign keys
SQL_CMD=$(cat <<EOT
$LDAP
$PROVIDER
$SERVER
$DOMAIN
$SERVER_DOMAIN
EOT
)
echo -e ${SQL_CMD}
echo -e ${SQL_CMD} | sqlite3 /data/db.sqlite
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
# vim:ts=4:sw=4:noet
#
# Creates a Python 3 virtual environment. The target directory can be passed
# as a parameter. The default path is '.venv' in the current directory.
dir="${1:-.venv}"
echo "Setup dir $dir"
set -e
if [ -d "${dir}" ]; then
echo >&2 "Directory '${dir}' already exists, exiting."
exit 1
fi
python3 -m venv "${dir}"
source "${dir}/bin/activate"
set +e
pip install -U pip setuptools wheel || true
#set -e
## vim:tabstop=4:noexpandtab
##
## Creates a Python 3 virtual environment. The target directory can be passed
## as a parameter. The default path is 'venv' in the current directory.
#
#dir="${1:-venv}"
#
#set -e
#if [ -d "${dir}" ]; then
# echo "Directory '${dir}' already exists, exiting." >&2
# exit 1
#fi
#python3 -m venv "${dir}"
#. "${dir}/bin/activate"
#
#set +e
#pip install -U pip setuptools || true
-8
View File
@@ -1,8 +0,0 @@
#!/usr/bin/env bash
set -e
# Setup
./setup
# Start
./.venv/scripts/flask.sh run --host=0.0.0.0 --port=4243
+260
View File
@@ -0,0 +1,260 @@
package main
import (
"bytes"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"testing"
)
// The environment mailu gives automx on novox when the fixtures were captured (files/mailu.env): the mail domain,
// the route name for both servers, the ports.
var capturedEnvironment = map[string]string{
"AMX_MAIL_DOMAINS": "novox.be",
"AMX_IMAP_ADDRESS": "mail.novox.be",
"AMX_IMAP_PORT": "143",
"AMX_SMTP_ADDRESS": "mail.novox.be",
"AMX_SMTP_PORT": "587",
"AUTOMX_PORT": "4243",
}
func getenvOf(m map[string]string) func(string) string {
return func(k string) string { return m[k] }
}
func testServer(t *testing.T) *httptest.Server {
t.Helper()
cfg, err := configFrom(getenvOf(capturedEnvironment))
if err != nil {
t.Fatal(err)
}
s := httptest.NewServer(newHandler(cfg))
t.Cleanup(s.Close)
return s
}
const captured = "testdata/captured"
// A captured request: its first line is the method and the URL the client used, then the headers sent, then after
// a blank line the body sent (CAPTURED, how the files were written).
type request struct {
method, url string
header [][2]string
body string
}
func readRequest(t *testing.T, stem string) request {
t.Helper()
raw, err := os.ReadFile(stem + ".request")
if err != nil {
t.Fatal(err)
}
head, body, hasBody := strings.Cut(string(raw), "\n\n")
if !hasBody {
head = strings.TrimSuffix(head, "\n")
}
lines := strings.Split(head, "\n")
method, target, ok := strings.Cut(lines[0], " ")
if !ok {
t.Fatalf("%s: first line %q is not METHOD URL", stem, lines[0])
}
r := request{method: method, url: target, body: body}
for _, l := range lines[1:] {
k, v, ok := strings.Cut(l, ": ")
if !ok {
t.Fatalf("%s: header line %q", stem, l)
}
r.header = append(r.header, [2]string{k, v})
}
return r
}
func readFile(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile(name)
if err != nil {
t.Fatal(err)
}
return string(b)
}
// A fixture asks for its body to differ only where DIFFERENCES says so: a stem with a .corrected.body is answered
// with that, and its Content-Length is the corrected body's.
func expectedBody(t *testing.T, stem string) (body string, corrected bool) {
if _, err := os.Stat(stem + ".corrected.body"); err == nil {
return readFile(t, stem+".corrected.body"), true
}
return readFile(t, stem+".body"), false
}
func TestEveryCapturedRequestIsAnsweredAsAutomxAnswered(t *testing.T) {
stems, _ := filepath.Glob(filepath.Join(captured, "*.request"))
if len(stems) < 17 {
t.Fatalf("%d captured requests found, want the 17 of issue 1630", len(stems))
}
srv := testServer(t)
client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
for _, f := range stems {
stem := strings.TrimSuffix(f, ".request")
t.Run(filepath.Base(stem), func(t *testing.T) {
rq := readRequest(t, stem)
u, err := url.Parse(rq.url)
if err != nil {
t.Fatal(err)
}
// The mesh's route reaches the program with the client's name and scheme; the program is told them in
// the forwarding headers (the Location of fixture 14 holds them).
req, err := http.NewRequest(rq.method, srv.URL+u.RequestURI(), strings.NewReader(rq.body))
if err != nil {
t.Fatal(err)
}
for _, h := range rq.header {
req.Header.Set(h[0], h[1])
}
req.Host = u.Host
req.Header.Set("X-Forwarded-Proto", u.Scheme)
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
got, _ := io.ReadAll(resp.Body)
if want := strings.TrimRight(readFile(t, stem+".status"), "\n"); resp.Status != want {
t.Errorf("status %q, captured %q", resp.Status, want)
}
wantBody, corrected := expectedBody(t, stem)
if !bytes.Equal(got, []byte(wantBody)) {
t.Errorf("body\n%q\nwant\n%q", got, wantBody)
}
want := map[string]string{}
for _, l := range strings.Split(strings.TrimRight(readFile(t, stem+".headers"), "\n"), "\n") {
if k, v, ok := strings.Cut(l, ": "); ok {
want[k] = v
}
}
if corrected {
want["Content-Length"] = strconv.Itoa(len(wantBody))
}
// Server names the Python program that answered (DIFFERENCES); Date changes on every request.
delete(want, "Server")
var extra []string
for k := range resp.Header {
if k == "Date" || k == "Server" {
continue
}
if _, ok := want[k]; !ok {
extra = append(extra, k)
}
}
sort.Strings(extra)
if len(extra) > 0 {
t.Errorf("headers the capture does not have: %v", extra)
}
for k, v := range want {
if g := resp.Header.Get(k); g != v {
t.Errorf("header %s: %q, captured %q", k, g, v)
}
}
})
}
}
// DIFFERENCES: the one change clause 9 of ADR 0480 allows to an answer is the IMAP server's name, written from
// AMX_IMAP_ADDRESS where automx wrote an empty one.
func TestTheOnlyCorrectedAnswerIsTheImapServersName(t *testing.T) {
corrected, _ := filepath.Glob(filepath.Join(captured, "*.corrected.body"))
if len(corrected) != 3 {
t.Fatalf("%d corrected answers, want the three Thunderbird answers (01, 02, 03): %v", len(corrected), corrected)
}
for _, c := range corrected {
orig := readFile(t, strings.TrimSuffix(c, ".corrected.body")+".body")
want := strings.Replace(orig, "<hostname />", "<hostname>mail.novox.be</hostname>", 1)
if strings.Contains(want, "<hostname />") || want == orig {
t.Fatalf("%s: the capture has no single empty hostname to correct", c)
}
if got := readFile(t, c); got != want {
t.Errorf("%s differs from the capture by more than the IMAP server's name", c)
}
}
}
func TestTheImapServerIsNamedFromTheImapAddress(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_IMAP_ADDRESS"] = "imap.example.test"
env["AMX_IMAP_PORT"] = "993"
cfg, err := configFrom(getenvOf(env))
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
newHandler(cfg).ServeHTTP(rec, httptest.NewRequest("GET", "/mail/config-v1.1.xml?emailaddress=a@novox.be", nil))
body := rec.Body.String()
if !strings.Contains(body, `<incomingServer type="imap"><hostname>imap.example.test</hostname><port>993</port>`) {
t.Errorf("incoming server not named from AMX_IMAP_ADDRESS and AMX_IMAP_PORT: %s", body)
}
if !strings.Contains(body, `<outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port>`) {
t.Errorf("outgoing server changed: %s", body)
}
}
func TestEveryDomainOfTheEnvironmentIsServed(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_MAIL_DOMAINS"] = "novox.be, example.test"
cfg, err := configFrom(getenvOf(env))
if err != nil {
t.Fatal(err)
}
h := newHandler(cfg)
for _, d := range []string{"novox.be", "example.test"} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest("GET", "/mail/config-v1.1.xml?emailaddress=a@"+d, nil))
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "<domain>"+d+"</domain>") {
t.Errorf("%s: %d %s", d, rec.Code, rec.Body.String())
}
}
}
// Failure is loud (issue 1631 AC2): a missing variable stops the program and its name is in the message.
func TestAMissingVariableStopsTheProgramNamingIt(t *testing.T) {
for name := range capturedEnvironment {
t.Run(name, func(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
delete(env, name)
if _, err := configFrom(getenvOf(env)); err == nil || !strings.Contains(err.Error(), name) {
t.Fatalf("configFrom without %s: %v", name, err)
}
var out bytes.Buffer
if code := run(getenvOf(env), &out, func(*http.Server) error { t.Fatal("served without its configuration"); return nil }); code == 0 || !strings.Contains(out.String(), name) {
t.Errorf("run without %s: exit %d, said %q", name, code, out.String())
}
})
}
}
func TestAPortThatIsNotANumberIsRefusedNamingIt(t *testing.T) {
env := map[string]string{}
for k, v := range capturedEnvironment {
env[k] = v
}
env["AMX_SMTP_PORT"] = "submission"
if _, err := configFrom(getenvOf(env)); err == nil || !strings.Contains(err.Error(), "AMX_SMTP_PORT") {
t.Fatalf("got %v", err)
}
}
+290
View File
@@ -0,0 +1,290 @@
// automx answers mail clients' autoconfiguration requests for the domains mailu serves: Thunderbird's
// /mail/config-v1.1.xml, Outlook's /autodiscover/autodiscover.xml and Apple's /mobileconfig. It replaces the
// Python automx2 image (novox/hq ADR 0480 clauses 1, 9, 10): standard library only, no database, every answer
// made from mailu's environment at the start, so a change of the domains, servers or ports reaches clients when
// the program is started again.
//
// The answers are those automx2 2021.6 gave, captured in testdata/captured (CAPTURED, issue 1630), byte for byte,
// but for the one thing testdata/captured/DIFFERENCES lists: the IMAP server is named, from AMX_IMAP_ADDRESS.
package main
import (
"encoding/xml"
"errors"
"fmt"
"html"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
)
// The environment, all required: a program that cannot say which servers clients use must not run (failure is
// loud). The names are those of mailu's files/mailu.env.
const (
envDomains = "AMX_MAIL_DOMAINS"
envImapHost = "AMX_IMAP_ADDRESS"
envImapPort = "AMX_IMAP_PORT"
envSmtpHost = "AMX_SMTP_ADDRESS"
envSmtpPort = "AMX_SMTP_PORT"
envListen = "AUTOMX_PORT"
)
type config struct {
domains []string
imapHost, smtpHost string
imapPort, smtpPort int
listen int
}
// configFrom reads the environment; it names every variable that is missing or wrong.
func configFrom(getenv func(string) string) (config, error) {
var c config
var problems []string
text := func(name string) string {
v := strings.TrimSpace(getenv(name))
if v == "" {
problems = append(problems, name+" is not set")
}
return v
}
port := func(name string) int {
v := text(name)
if v == "" {
return 0
}
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 65535 {
problems = append(problems, fmt.Sprintf("%s is %q, not a port number", name, v))
return 0
}
return n
}
// automx2's add-domains split the list at commas and spaces.
for _, d := range strings.FieldsFunc(text(envDomains), func(r rune) bool { return r == ',' || r == ' ' }) {
c.domains = append(c.domains, strings.ToLower(d))
}
c.imapHost, c.imapPort = text(envImapHost), port(envImapPort)
c.smtpHost, c.smtpPort = text(envSmtpHost), port(envSmtpPort)
c.listen = port(envListen)
if len(problems) > 0 {
return config{}, errors.New(strings.Join(problems, "; "))
}
return c, nil
}
// What automx2 sent (Werkzeug), copied from the capture.
const (
htmlType = "text/html; charset=utf-8"
xmlType = "application/xml"
notFound = `<!doctype html>
<html lang=en>
<title>404 Not Found</title>
<h1>Not Found</h1>
<p>The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.</p>
`
notAllowed = `<!doctype html>
<html lang=en>
<title>405 Method Not Allowed</title>
<h1>Method Not Allowed</h1>
<p>The method is not allowed for the requested URL.</p>
`
internalError = `<!doctype html>
<html lang=en>
<title>500 Internal Server Error</title>
<h1>Internal Server Error</h1>
<p>The server encountered an internal error and was unable to complete your request. Either the server is overloaded or there is an error in the application.</p>
`
badRequest = `<!doctype html>
<html lang=en>
<title>400 Bad Request</title>
<h1>Bad Request</h1>
<p>The browser (or proxy) sent a request that this server could not understand.</p>
`
index = `<html><body>Show Thunderbird-style <a href="/mail/config-v1.1.xml?emailaddress=abc@example.com">XML configuration</a> for abc@example.com.</body></html>`
)
func reply(w http.ResponseWriter, status int, contentType, body string) {
w.Header().Set("Content-Type", contentType)
if status != http.StatusNoContent {
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
}
w.WriteHeader(status)
if status != http.StatusNoContent {
io.WriteString(w, body)
}
}
func missing(w http.ResponseWriter, argument string) {
reply(w, http.StatusBadRequest, htmlType, fmt.Sprintf("Missing request argument %q", argument))
}
type handler struct{ config }
func newHandler(c config) http.Handler { return handler{c} }
func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
var allow string
switch r.URL.Path {
case "/", "/mail/config-v1.1.xml", "/mobileconfig/":
allow = "GET, HEAD, OPTIONS"
case "/autodiscover/autodiscover.xml":
allow = "POST, OPTIONS"
case "/mobileconfig":
h.redirectToSlash(w, r)
return
default:
reply(w, http.StatusNotFound, htmlType, notFound)
return
}
if r.Method == http.MethodOptions {
w.Header().Set("Allow", allow)
reply(w, http.StatusOK, htmlType, "")
return
}
if !strings.Contains(", "+allow+",", ", "+r.Method+",") {
w.Header().Set("Allow", allow)
reply(w, http.StatusMethodNotAllowed, htmlType, notAllowed)
return
}
switch r.URL.Path {
case "/":
reply(w, http.StatusOK, htmlType, index)
case "/mail/config-v1.1.xml":
h.byAddress(w, r.URL.Query().Get("emailaddress"), r.URL.Query().Has("emailaddress"), "emailaddress", h.thunderbird)
case "/mobileconfig/":
h.byAddress(w, r.URL.Query().Get("emailaddress"), r.URL.Query().Has("emailaddress"), "emailaddress", h.apple)
case "/autodiscover/autodiscover.xml":
h.outlook(w, r)
}
}
// byAddress is the part the three protocols share: the address is required, and a domain that is not served is
// answered with no content.
func (h handler) byAddress(w http.ResponseWriter, address string, present bool, argument string, answer func(http.ResponseWriter, int, string)) {
if !present {
missing(w, argument)
return
}
domain := strings.ToLower(address[strings.LastIndex(address, "@")+1:])
for i, d := range h.domains {
if strings.Contains(address, "@") && d == domain {
answer(w, i, d)
return
}
}
reply(w, http.StatusNoContent, htmlType, "")
}
// thunderbird is Mozilla's autoconfiguration, config-v1.1. The provider id is automx2's: the provider rows were
// numbered 1, 11, 21 ... by add-domains.
func (h handler) thunderbird(w http.ResponseWriter, i int, domain string) {
d := html.EscapeString(domain)
server := func(kind, host string, port int) string {
tag := map[string]string{"imap": "incomingServer", "smtp": "outgoingServer"}[kind]
return fmt.Sprintf(`<%s type="%s"><hostname>%s</hostname><port>%d</port><socketType>STARTTLS</socketType>`+
`<username>%%EMAILADDRESS%%</username><authentication>password-cleartext</authentication></%s>`,
tag, kind, html.EscapeString(host), port, tag)
}
body := fmt.Sprintf(`<clientConfig version="1.1"><emailProvider id="automx2-%d"><identity /><domain>%s</domain>`+
`<displayName>%s</displayName><displayShortName>%s</displayShortName>%s%s</emailProvider></clientConfig>`,
1+10*i, d, d, d, server("imap", h.imapHost, h.imapPort), server("smtp", h.smtpHost, h.smtpPort))
reply(w, http.StatusOK, xmlType, body)
}
// apple and outlook answer a served domain as automx2 did: with an error (DIFFERENCES, issue 1691). They are
// kept as captured here, so the move to a Go program changes the answer only where clause 9 says.
func (h handler) apple(w http.ResponseWriter, _ int, _ string) {
reply(w, http.StatusBadRequest, htmlType, badRequest)
}
// redirectToSlash is Werkzeug's strict-slash redirect, to the name the client used.
func (h handler) redirectToSlash(w http.ResponseWriter, r *http.Request) {
scheme := "http"
if p := strings.TrimSpace(strings.Split(r.Header.Get("X-Forwarded-Proto"), ",")[0]); p != "" {
scheme = p
}
host := r.Host
if f := strings.TrimSpace(strings.Split(r.Header.Get("X-Forwarded-Host"), ",")[0]); f != "" {
host = f
}
target := scheme + "://" + host + "/mobileconfig/"
if r.URL.RawQuery != "" {
target += "?" + r.URL.RawQuery
}
w.Header().Set("Location", target)
e := html.EscapeString(target)
reply(w, http.StatusPermanentRedirect, htmlType, fmt.Sprintf(`<!doctype html>
<html lang=en>
<title>Redirecting...</title>
<h1>Redirecting...</h1>
<p>You should be redirected automatically to the target URL: <a href="%s">%s</a>. If not, click the link.
`, e, e))
}
// outlook reads the request's EMailAddress anywhere in the document, whatever the namespace. A body that is not
// one XML document is automx2's 500.
func (h handler) outlook(w http.ResponseWriter, r *http.Request) {
dec := xml.NewDecoder(io.LimitReader(r.Body, 1<<20))
var address string
var found, elements, inAddress = false, 0, false
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
reply(w, http.StatusInternalServerError, htmlType, internalError)
return
}
switch t := tok.(type) {
case xml.StartElement:
elements++
if t.Name.Local == "EMailAddress" && !found {
inAddress = true
}
case xml.CharData:
if inAddress {
address += string(t)
}
case xml.EndElement:
if inAddress && t.Name.Local == "EMailAddress" {
inAddress, found = false, true
}
}
}
if elements == 0 {
reply(w, http.StatusInternalServerError, htmlType, internalError)
return
}
h.byAddress(w, strings.TrimSpace(address), found, "EMailAddress", func(w http.ResponseWriter, _ int, _ string) {
reply(w, http.StatusInternalServerError, htmlType, internalError)
})
}
// run is the program: configuration from the environment, then serving. A configuration that cannot be read is
// said, with the variables' names, and the exit code is not zero.
func run(getenv func(string) string, out io.Writer, serve func(*http.Server) error) int {
c, err := configFrom(getenv)
if err != nil {
fmt.Fprintf(out, "automx: %v\n", err)
return 1
}
srv := &http.Server{
Addr: ":" + strconv.Itoa(c.listen),
Handler: newHandler(c),
ReadHeaderTimeout: 10 * time.Second,
}
if err := serve(srv); err != nil {
fmt.Fprintf(out, "automx: %v\n", err)
return 1
}
return 0
}
func main() {
os.Exit(run(os.Getenv, os.Stderr, func(s *http.Server) error { return s.ListenAndServe() }))
}
+202
View File
@@ -0,0 +1,202 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// The digest route-proxy's `trust` artifact runs: one alpine image is held for both (ADR 0480 clause 3). It was
// read from mesh-controller's modules/route-proxy/module.json and is the one hello-web runs.
const alpineDigest = "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
type manifest struct {
Tools []string `json:"tools"`
Resources []map[string]any `json:"resources"`
Listens []struct {
Name string `json:"name"`
Port int `json:"port"`
} `json:"listens"`
Data struct {
Own []struct {
ID string `json:"id"`
Path string `json:"path"`
} `json:"own"`
} `json:"data"`
Build map[string]any `json:"build"`
}
func readManifest(t *testing.T) manifest {
t.Helper()
var m manifest
if err := json.Unmarshal([]byte(readFile(t, "../../module.json")), &m); err != nil {
t.Fatal(err)
}
return m
}
func (m manifest) resource(id string) map[string]any {
for _, r := range m.Resources {
if r["id"] == id {
return r
}
}
return nil
}
func (m manifest) artifact(name string) map[string]any {
arts, _ := m.Build["artifacts"].([]any)
for _, a := range arts {
if a, _ := a.(map[string]any); a["name"] == name {
return a
}
}
return nil
}
func strs(v any) []string {
var out []string
l, _ := v.([]any)
for _, e := range l {
out = append(out, e.(string))
}
return out
}
// Clause 2: a Go bundle from a package of the module's own go.mod, stamped for alpine; an archive resource unpacks
// it; the container mounts that read-only at /opt/mesh, calls the program there and names the archive in restart-on.
func TestAutomxIsAGoBundleForAlpineMountedAtOptMeshAndRestartedWithIt(t *testing.T) {
m := readManifest(t)
b := m.artifact("automx")
if b == nil || b["kind"] != "bundle" || b["language"] != "go" || b["system"] != "alpine" || b["from"] != "cmd/automx" || b["binary"] != "automx" {
t.Fatalf("artifact automx is %v, want a Go bundle, system alpine, from cmd/automx, binary automx", b)
}
if _, loads := b["loads"]; loads {
t.Errorf("a program run in a container is not loaded by the tool runner: %v", b["loads"])
}
goMod := readFile(t, "../../go.mod")
if !strings.HasPrefix(goMod, "module mailu\n") {
t.Errorf("the module's go.mod is %q; the bundle is a package of it", goMod)
}
if _, err := os.Stat(filepath.Join("..", "..", filepath.FromSlash(b["from"].(string)), "main.go")); err != nil {
t.Errorf("the bundle's package is not in the module's directory: %v", err)
}
unpack := m.resource("program-automx")
if unpack == nil || unpack["type"] != "archive" || unpack["artifact"] != "automx" || unpack["unknown"] != "program" {
t.Fatalf("resource program-automx is %v, want an archive of the bundle", unpack)
}
where, _ := unpack["path"].(string)
c := m.resource("automx")
if c == nil || c["type"] != "container" {
t.Fatalf("no container automx: %v", c)
}
if got := strs(c["volumes"]); len(got) != 1 || got[0] != where+":/opt/mesh:ro" {
t.Errorf("volumes %v, want the archive's place %q mounted read-only at /opt/mesh", got, where)
}
if got := strs(c["args"]); len(got) != 1 || got[0] != "/opt/mesh/automx" {
t.Errorf("args %v, want the program at /opt/mesh/automx", got)
}
if !contains(strs(c["restart-on"]), "program-automx") {
t.Errorf("restart-on %v does not name the archive: a new program would leave the old one running", c["restart-on"])
}
if !contains(strs(c["restart-on"]), "config-env") {
t.Errorf("restart-on %v does not name the environment file the program reads at its start", c["restart-on"])
}
}
// Clause 3: an upstream artifact pinned by digest, the one route-proxy's trust artifact runs; never an image
// built from a Dockerfile (clause 11), and no Dockerfile or base to build one on.
func TestAutomxRunsTheMirroredAlpineImageAndBuildsNoImage(t *testing.T) {
m := readManifest(t)
c := m.resource("automx")
base := m.artifact(c["artifact"].(string))
if base == nil || base["kind"] != "upstream" || base["from"] != alpineDigest {
t.Errorf("the container runs artifact %v, want an upstream artifact %s", base, alpineDigest)
}
if _, has := c["image"]; has {
t.Errorf("the container names an image %v, fetched from a registry on the machine", c["image"])
}
for _, a := range m.Build["artifacts"].([]any) {
if a.(map[string]any)["kind"] == "image" {
t.Errorf("artifact %v is an image built from a Dockerfile (ADR 0452)", a)
}
}
if _, has := m.Build["on"]; has {
t.Errorf("build.on %v: nothing is built on a base image any more", m.Build["on"])
}
for _, gone := range []string{"../../automx/Dockerfile", "../../automx"} {
if _, err := os.Stat(gone); err == nil {
t.Errorf("%s is still there", gone)
}
}
}
// Clause 1: the container stays as it was for what watches it.
func TestAutomxKeepsItsContainerNameNetworkPortAndEnvironment(t *testing.T) {
m := readManifest(t)
c := m.resource("automx")
if c["name"] != "mailu-automx" || c["network"] != "mailu" {
t.Errorf("container %v on network %v, want mailu-automx on mailu", c["name"], c["network"])
}
if got := strs(c["ports"]); len(got) != 1 || got[0] != "4243" {
t.Errorf("ports %v, want 4243", got)
}
if got := strs(c["env-file"]); len(got) != 1 || got[0] != "${dir:state}/mailu.env" {
t.Errorf("env-file %v, want mailu's environment only", got)
}
if !strings.Contains("\n"+readFile(t, "../../files/mailu.env"), "\nAUTOMX_PORT=4243\n") {
t.Errorf("mailu.env does not say AUTOMX_PORT=4243")
}
for _, l := range m.Listens {
if l.Name == "autoconfig" && l.Port == 4243 {
return
}
}
t.Errorf("the autoconfig endpoint is not on 4243: %v", m.Listens)
}
// Clause 10: a database nobody reads is gone from the declaration (the file stays on disk), and the container says
// whether it answers with one captured request and its captured answer.
func TestAutomxKeepsNoDatabaseAndSaysWhetherItAnswers(t *testing.T) {
m := readManifest(t)
if m.resource("data-automx") != nil {
t.Errorf("the data-automx directory is still declared")
}
for _, it := range m.Data.Own {
if strings.Contains(it.ID, "automx") || strings.Contains(it.Path, "data-automx") {
t.Errorf("data item %v is still declared", it)
}
}
for _, tool := range m.Tools {
if tool == "automx_rows" {
t.Errorf("the tool automx_rows reads a database that no longer exists (issue 1700)")
}
}
h, _ := m.resource("automx")["health"].(map[string]any)
if h["kind"] != "http" || h["endpoint"] != "autoconfig" {
t.Fatalf("health %v, want an http check of the autoconfig endpoint", h)
}
// The request is captured fixture 06 and the answer its status and body.
rq := readRequest(t, filepath.Join(captured, "06-autoconfig-root"))
if u := strings.SplitN(rq.url, "/", 4); "/"+u[len(u)-1] != h["path"] || rq.method != "GET" {
t.Errorf("health path %v is not captured request %s %s", h["path"], rq.method, rq.url)
}
if st := readFile(t, filepath.Join(captured, "06-autoconfig-root.status")); !strings.HasPrefix(st, "200 ") || h["status"] != float64(200) {
t.Errorf("health status %v, captured %q", h["status"], st)
}
if body, _ := h["body"].(string); body == "" || !strings.Contains(readFile(t, filepath.Join(captured, "06-autoconfig-root.body")), body) {
t.Errorf("health body %q is not in the captured answer", h["body"])
}
}
func contains(l []string, s string) bool {
for _, e := range l {
if e == s {
return true
}
}
return false
}
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
@@ -0,0 +1 @@
<clientConfig version="1.1"><emailProvider id="automx2-1"><identity /><domain>novox.be</domain><displayName>novox.be</displayName><displayShortName>novox.be</displayShortName><incomingServer type="imap"><hostname>mail.novox.be</hostname><port>143</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></incomingServer><outgoingServer type="smtp"><hostname>mail.novox.be</hostname><port>587</port><socketType>STARTTLS</socketType><username>%EMAILADDRESS%</username><authentication>password-cleartext</authentication></outgoingServer></emailProvider></clientConfig>
+12 -1
View File
@@ -14,6 +14,13 @@ Each capture is four files with the same stem:
The files were written from the tool's answers by a script (not by retyping each by hand); the tool returns JSON, so
a later capture that wants exact bytes needs a tool that writes the answer to a file.
A fixture is not the bytes on the wire in four ways (issue 1701):
- Date is left out of .headers (it changes on every request).
- .headers are sorted by name.
- .status holds the tool's status line without the protocol (the tool reported "HTTP/1.1" apart).
- Header names are as the tool's Go HTTP client gives them (canonical case, after the mesh's route), not as
automx wrote them.
What the answers show (read, not guessed):
- GET /mail/config-v1.1.xml?emailaddress=... answers 200 application/xml on all three names; same body on each.
- The incoming (IMAP) server has an EMPTY hostname (<hostname />), port 143, STARTTLS. The outgoing server is
@@ -27,4 +34,8 @@ What the answers show (read, not guessed):
- GET /mobileconfig?emailaddress=... redirects (308) to /mobileconfig/?emailaddress=...; for the served domain that
answers 400 with no profile; unknown domain 204; no emailaddress 400.
- GET / on autoconfig answers 200 with a small HTML page linking to an example address.
- Not captured: the database rows (no mesh tool reads /data/db.sqlite; issue 1630 AC4).
- The database rows behind the answers were read once with the tool automx_rows (mailu, issue 1630 AC4), which
ran sqlite3 read-only inside mailu-automx. The tool is retired with the database (ADR 0480 clause 10, issue
1700), so nothing reads the rows any more; the answers above are what they gave.
- The Go program that replaced automx2 answers these requests as they are captured here, but for what
DIFFERENCES lists.
+25
View File
@@ -0,0 +1,25 @@
How the Go program (modules/mailu/cmd/automx) answers differently from the automx2 2021.6 answers captured here
(issue 1631, ADR 0480 clause 9). automx_test.go answers every .request and compares with the capture; this is the
whole list of where the two differ.
1. The IMAP server's name (the one correction clause 9 allows). automx2 answered <hostname /> for the incoming
server because its setup script read AMX_IMAP_SERVER, which mailu's environment does not set (it sets
AMX_IMAP_ADDRESS). The program names it from AMX_IMAP_ADDRESS (and the port from AMX_IMAP_PORT).
Fixtures 01, 02 and 03 each have a <stem>.corrected.body: the capture with <hostname /> replaced by
<hostname>mail.novox.be</hostname> and nothing else (a test checks that). Their Content-Length is that file's
size, 634, not the captured 612.
2. The Server header. The capture has "Server: Werkzeug/3.1.9 Python/3.9.5". The program sends no Server header, and
Date, which the captured .headers leave out, is sent as net/http sends it.
Not changed, though issue 1691 calls them faults; they are kept as captured because clause 9 allows only the first
correction, and their cause is not known from the capture:
- Outlook's POST for a served domain (fixture 08) answers 500.
- Apple's /mobileconfig/ for a served domain (fixture 15) answers 400 with no profile.
Whether automx2 failed there because of the empty IMAP name is a guess; it can be tested only against the
running program after the delivery. Issue 1691 stays open for them.
Assumptions the fixtures do not settle (answers to requests nobody captured): a domain is compared in lower case;
an address without "@" is an unknown domain (204); HEAD is answered as GET and OPTIONS with the Allow header and
no body; the redirect of /mobileconfig takes its scheme and host from X-Forwarded-Proto and X-Forwarded-Host when
present (the route sends the client's, as fixture 14's Location shows) and from the request otherwise.
+3
View File
@@ -0,0 +1,3 @@
module mailu
go 1.22
+34 -24
View File
@@ -71,8 +71,7 @@
"mailu_list_domains",
"mailu_read_mail",
"mailu_search_mail",
"mailu_health",
"automx_rows"
"mailu_health"
],
"emits": [
"user.created",
@@ -216,12 +215,6 @@
"class": "rebuildable",
"why": "certificates, issued again"
},
{
"id": "automx",
"path": "${dir:data-automx}",
"class": "rebuildable",
"why": "client autoconfiguration, written again"
},
{
"id": "fetchmail",
"path": "${dir:data-fetchmail}",
@@ -308,12 +301,6 @@
"mode": "0700",
"unknown": "program"
},
{
"id": "data-automx",
"type": "directory",
"mode": "0700",
"unknown": "program"
},
{
"id": "config-env",
"type": "file",
@@ -681,11 +668,25 @@
"takes-config": ["domain", "proxy-address", "sitename", "website", "url", "container"],
"merge": "json"
},
{
"id": "program-automx",
"type": "archive",
"path": "${dir:state}/automx",
"artifact": "automx",
"unknown": "program"
},
{
"id": "automx",
"type": "container",
"name": "mailu-automx",
"artifact": "automx",
"artifact": "alpine",
"health": {
"kind": "http",
"endpoint": "autoconfig",
"path": "/",
"status": 200,
"body": "Show Thunderbird-style"
},
"network": "mailu",
"env-file": [
"${dir:state}/mailu.env"
@@ -694,17 +695,18 @@
"4243"
],
"volumes": [
"${dir:data-automx}:/data"
"${dir:state}/automx:/opt/mesh:ro"
],
"args": [
"/opt/mesh/automx"
],
"restart-on": [
"program-automx",
"config-env"
]
}
],
"build": {
"on": [
{
"arg": "PYTHON_BASE",
"image": "python@sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"
}
],
"artifacts": [
{
"name": "code",
@@ -731,8 +733,16 @@
},
{
"name": "automx",
"kind": "image",
"from": "automx/Dockerfile"
"kind": "bundle",
"language": "go",
"system": "alpine",
"from": "cmd/automx",
"binary": "automx"
},
{
"name": "alpine",
"kind": "upstream",
"from": "alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"
},
{
"name": "files",
-103
View File
@@ -1,103 +0,0 @@
// What automx_rows reads (novox/hq issue 1630): the rows of automx2's database as stored, through a read-only
// sqlite3 inside its container. The database here is a real SQLite file made by the test with the tables
// automx/files/setup-db creates and the rows add-domains writes — with the IMAP server's name empty, the case
// the live instance is in. The fake container runs each command's statement against that database and prints
// it the way the image's sqlite3 does in list mode (one line per row, cells apart by "|", nothing for no rows); that sqlite3
// refuses `-json` ("sqlite3: Error: unknown option: -json", captured from novox/mailu.automx_rows, issue 1630). Run against the compiled module.
import { test } from "node:test";
import assert from "node:assert/strict";
import { DatabaseSync } from "node:sqlite";
import { readAutomx, readCommand, selectOf, parseRows, AUTOMX_DATABASE } from "../dist/automx.js";
function database(imapName: string): DatabaseSync {
const db = new DatabaseSync(":memory:");
db.exec(`
CREATE TABLE provider(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, short_name TEXT NOT NULL);
CREATE TABLE server(id INT PRIMARY KEY NOT NULL, prio INT NOT NULL DEFAULT 10, name TEXT NOT NULL, port INT NOT NULL,
type TEXT NOT NULL, socket_type TEXT NOT NULL, user_name TEXT NOT NULL, authentication TEXT NOT NULL);
CREATE TABLE domain(id INT PRIMARY KEY NOT NULL, name TEXT NOT NULL, provider_id INT NOT NULL, ldapserver_id INT NULL);
CREATE TABLE server_domain(server_id INT NOT NULL, domain_id INT NOT NULL);
INSERT INTO provider VALUES (1, 'example.org', 'example.org');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES (2, 143, 'imap', '${imapName}', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO server(id, port, type, name, socket_type, user_name, authentication)
VALUES (3, 587, 'smtp', 'mail.example.org', 'STARTTLS', '%EMAILADDRESS%', 'password-cleartext');
INSERT INTO domain VALUES (5, 'example.org', 1, NULL);
INSERT INTO server_domain VALUES (2, 5), (3, 5);
`);
return db;
}
function container(db: DatabaseSync, asked: string[][] = []) {
return async (_name: string, command: string[]) => {
asked.push(command);
if (command.includes("-json")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "sqlite3: Error: unknown option: -json\n" });
const rows = db.prepare(command[command.length - 1]).all() as unknown as Record<string, string>[];
return rows.length === 0 ? "" : rows.map((r) => Object.values(r).join("|")).join("\n") + "\n";
};
}
test("the rows come back as stored, and an empty server name is said", async () => {
const answer = await readAutomx(container(database("")), "mailu-automx");
assert.equal(answer.database, AUTOMX_DATABASE);
assert.deepEqual(answer.unreadable, {});
assert.deepEqual(answer.tables.provider, [{ id: 1, name: "example.org", short_name: "example.org" }]);
assert.deepEqual(answer.tables.domain, [{ id: 5, name: "example.org", provider_id: 1, ldapserver_id: null }]);
assert.deepEqual(answer.tables.server_domain, [{ server_id: 2, domain_id: 5 }, { server_id: 3, domain_id: 5 }]);
assert.deepEqual(
answer.tables.server.map((s) => [s.id, s.type, s.name, s.port]),
[[2, "imap", "", 143], [3, "smtp", "mail.example.org", 587]],
);
assert.deepEqual(answer.empty_names, [{ id: 2, type: "imap" }]);
});
test("a named IMAP server is not reported empty", async () => {
const answer = await readAutomx(container(database("mail.example.org")), "mailu-automx");
assert.deepEqual(answer.empty_names, []);
});
test("every command opens the file read-only and runs one fixed SELECT", async () => {
const asked: string[][] = [];
await readAutomx(container(database("x"), asked), "mailu-automx");
assert.equal(asked.length, 4);
for (const c of asked) {
assert.deepEqual(c.slice(0, 3), ["sqlite3", "-readonly", AUTOMX_DATABASE]);
assert.match(c[3], /^SELECT /);
assert.equal(c.length, 4);
assert.ok(!c.includes("-json"));
}
assert.deepEqual(readCommand("provider"), ["sqlite3", "-readonly", AUTOMX_DATABASE, selectOf("provider")]);
});
test("a table that cannot be read is said and does not hide the others", async () => {
const db = database("x");
const exec = async (name: string, command: string[]) => {
if (command[3].includes("FROM domain ")) throw Object.assign(new Error("Command failed: docker exec"), { stderr: "Error: unable to open database file\nmore" });
return container(db)(name, command);
};
const answer = await readAutomx(exec, "mailu-automx");
assert.deepEqual(answer.unreadable, { domain: "Error: unable to open database file" });
assert.equal(answer.tables.provider.length, 1);
assert.equal(answer.tables.domain, undefined);
});
test("a NULL, an empty text, a number and a text holding a bar or a newline are read apart", () => {
const hex = (t: string) => Buffer.from(t).toString("hex").toUpperCase();
const line = [`integer:${hex("5")}`, `text:${hex("a|b\nc")}`, "text:", "null:"].join("|");
const rows = parseRows("domain", line + "\n");
assert.deepEqual(rows, [{ id: 5, name: "a|b\nc", provider_id: "", ldapserver_id: null }]);
});
test("output that is not cells is an unreadable table, not a guess", async () => {
const exec = async () => "Error: something else\n";
const answer = await readAutomx(exec, "mailu-automx");
assert.equal(Object.keys(answer.unreadable).length, 4);
});
test("the image's refusal of -json is said as the table's reason", async () => {
const exec = async (_n: string, command: string[]) => container(database("x"))("", ["sqlite3", "-json", ...command.slice(1)]);
const answer = await readAutomx(exec, "mailu-automx");
assert.deepEqual(answer.unreadable.domain, "sqlite3: Error: unknown option: -json");
});
+1 -2
View File
@@ -135,7 +135,6 @@ test("the health tool is served without the admin API, on the smtp container it
await import("../dist/tools/index.js");
const mailu = collectTools({ MESH_MAILU_SMTP_CONTAINER: "mailu-smtp" }).find((m) => m.module === "mailu");
const names = (mailu?.tools ?? []).map((t) => t.name);
assert.deepEqual(names, ["mailu_health", "automx_rows"]);
assert.deepEqual(names, ["mailu_health"]);
assert.match(mailu!.tools[0].description, /inside mailu-smtp/);
assert.match(mailu!.tools[1].description, /read-only, inside mailu-automx/);
});
+1 -19
View File
@@ -4,7 +4,6 @@
import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools";
import { MailuClient } from "../client.js";
import { readAutomx } from "../automx.js";
import { dockerExec, mailFlows, DEFERRED_BOUND_SECONDS, PUBLIC_NAME, SPREAD_DOMAINS } from "../health.js";
/**
@@ -27,23 +26,6 @@ export function getHealthTool(env: NodeJS.ProcessEnv): ToolDefinition {
};
}
/**
* What automx holds, as stored (novox/hq issue 1630): the rows behind the answers it gives mail clients. It needs
* no admin API, only the automx container, so like the health tool it is served without the API.
*/
export function getAutomxTool(env: NodeJS.ProcessEnv): ToolDefinition {
const container = env.MESH_MAILU_AUTOMX_CONTAINER || "mailu-automx";
return {
name: "automx_rows",
description:
`Read the autoconfiguration module's database, as stored, read-only, inside ${container}: its provider, ` +
"server, domain and server_domain rows, and the servers whose name is empty (a mail client is then told " +
"an empty host name). Writes nothing.",
input: {},
run: async () => readAutomx(dockerExec(), container),
};
}
export function getMailuTools(mailu: MailuClient): ToolDefinition[] {
return [
{
@@ -191,5 +173,5 @@ registerModuleTools("mailu", (env) => {
} catch {
tools = [];
}
return [...tools, getHealthTool(env), getAutomxTool(env)];
return [...tools, getHealthTool(env)];
});
+1 -1
View File
@@ -8,5 +8,5 @@
"skipLibCheck": true,
"noEmit": true
},
"include": ["client.ts", "health.ts", "automx.ts", "index.ts", "tools/index.ts"]
"include": ["client.ts", "health.ts", "index.ts", "tools/index.ts"]
}
@@ -17,6 +17,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
// ForgeAlias is the ssh name the forge-account provision sets up in the account's ssh files (ADR 0409 §1); the
// remote of a fetch is ssh://<it>/<owner>/<repository>.git.
const ForgeAlias = "mesh-forge"
@@ -37,6 +39,18 @@ type FetchResult struct {
Workspace string `json:"workspace"`
Commit string `json:"commit"`
Modules []string `json:"modules"`
// The merge preview (ADR 0484 clause 3), only when the call gave a base: the trunk commit, the merge commit of the
// head onto it, the paths the merge changes, and the tree of each module directory (a directory holding a
// module.json) in the merge. A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the
// files, and the workspace is left at the trunk commit.
Base string `json:"base,omitempty"`
Merge string `json:"merge,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
// Beside are the sibling repositories placed as plain directories under the workspace's beside directory, which
// build_go_test hands to the tests as MESH_CHECK_BESIDE (ADR 0484 clause 4).
Beside []Sibling `json:"beside,omitempty"`
}
// fetchTool is the fetch step tool.
@@ -52,6 +66,12 @@ func (m *Module) fetchTool() StepTool {
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
"beside": map[string]any{"type": "array", "description": "optional sibling repositories the tests read, each placed as a plain directory (no symlink) under the workspace's beside/<name>, which build_go_test gives the tests as MESH_CHECK_BESIDE",
"items": map[string]any{"type": "object", "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
}, "required": []string{"repository", "commit"}}},
},
"required": []string{"repository", "commit"},
},
@@ -64,6 +84,14 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
base, _ := args["base"].(string)
if base != "" && !commitOK.MatchString(base) {
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
}
siblings, err := siblingsArg(args["beside"])
if err != nil {
return nil, err
}
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
@@ -74,14 +102,27 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
if err := os.MkdirAll(tree, 0o700); err != nil {
return nil, err
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref); err != nil {
remote := m.remote(repository)
if err := buildstep.FetchCommit(ctx, m.Run, tree, remote, commit, ref, m.FetchOptions...); err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit}
if base != "" {
if err := m.merge(ctx, &res, tree, remote, base, commit); err != nil {
return nil, err
}
}
if res.Beside, err = m.placeSiblings(ctx, root, siblings); err != nil {
return nil, err
}
dirs, err := m.goModDirs(ctx, tree)
if err != nil {
return nil, err
}
res.Modules = dirs
if base != "" && res.Merge == "" {
return res, nil // a conflict: the tree is at the trunk, nothing of the head is built or tested
}
for _, d := range dirs {
if _, err := os.Stat(filepath.Join(tree, d, "vendor", "modules.txt")); err == nil {
continue // a module that vendors resolves nothing
@@ -93,7 +134,7 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
}
}
}
return FetchResult{Workspace: id, Commit: commit, Modules: dirs}, nil
return res, nil
}
// goModDirs are the directories of the tree that hold a go.mod, relative to it, in order.
@@ -104,7 +145,7 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
}
dirs := []string{}
for _, l := range strings.Split(r.Stdout, "\n") {
if l = strings.TrimSpace(l); l != "" {
if l = strings.TrimSpace(l); l != "" && !inTestdata(l) {
dirs = append(dirs, filepath.ToSlash(filepath.Dir(l)))
}
}
@@ -112,6 +153,17 @@ func (m *Module) goModDirs(ctx context.Context, tree string) ([]string, error) {
return dirs, nil
}
// inTestdata says a path lies under a directory named testdata, whose go.mod files are fixtures Go ignores, not modules
// of the tree (issue 1810).
func inTestdata(path string) bool {
for _, seg := range strings.Split(filepath.ToSlash(path), "/") {
if seg == "testdata" {
return true
}
}
return false
}
// goEnv is what a go command of a step is given beyond the base environment: the installed go alone (GOTOOLCHAIN=local, ADR 0477 §1: no toolchain is ever downloaded),
// modules resolved by their go.mod and go.sum, the module's own caches, the mesh's own modules fetched directly (by
// git, through the account's rewrite of the forge's address to ssh), and nothing that asks a question. For a directory
@@ -30,7 +30,8 @@ type fakeStep struct {
gomods string
gate chan struct{} // when set, the first git command waits for it
goRule func(line string) forgekey.Ran
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
vendors []string // directories (relative to the tree) that get a vendor/modules.txt at checkout
rule func(line string) (forgekey.Ran, bool) // answers a command before the defaults do
}
func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
@@ -41,6 +42,11 @@ func (f *fakeStep) run(ctx context.Context, env []string, name string, args ...s
if f.gate != nil && strings.Contains(line, " init ") {
<-f.gate
}
if f.rule != nil {
if r, ok := f.rule(line); ok {
return r
}
}
switch {
case name == "git" && strings.Contains(line, " rev-parse HEAD"):
return forgekey.Ran{Stdout: commitSHA + "\n"}
@@ -0,0 +1,27 @@
package main
import (
"context"
"reflect"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// 1810/AC1: the go.mod files under a testdata directory are fixtures Go itself ignores, not modules of the tree, so
// the build modules neither test nor digest them (a repository holding this module held one whose test reads
// MESH_CHECK_BESIDE, which failed every check of it).
func TestGoModDirsLeaveOutTheFixturesUnderTestdata(t *testing.T) {
m := &Module{}
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
return forgekey.Ran{Stdout: "checks/deps/go.mod\ngo.mod\nmodules/x/cmd/x/testdata/repos/beside/go.mod\nmodules/x/go.mod\ntestdata/go.mod\n"}
}
got, err := m.goModDirs(context.Background(), t.TempDir())
if err != nil {
t.Fatal(err)
}
want := []string{".", "checks/deps", "modules/x"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("go.mod directories = %v, want %v", got, want)
}
}
@@ -60,7 +60,7 @@ func (m *Module) integration(ctx context.Context, workspace, tree string, packag
return nil, fmt.Errorf("the integration level's bus: %w", err)
}
defer stopBus()
env := []string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}
env := append([]string{"MESH_TEST_POSTGRES=" + postgres, "MESH_TEST_NATS=" + nats}, m.besideEnv(workspace)...)
// Compiling counts as setup: the tests are built once, here, with the addresses a TestMain may read, and the run
// below finds them built. Whether they compile is for the run to say; its failure is not read here.
for _, d := range dirs {
@@ -37,6 +37,10 @@ type Module struct {
// both outside the account's home (ADR 0466 §1); Home is the account's home.
Run forgekey.Runner
Home, Work, Cache string
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
RemoteBase string
FetchOptions []buildstep.Option
// HTTP fetches the files an archive artifact is pinned to (url and sha256); a client of its own with a fifteen-minute
// timeout when nil. A test serves its own.
HTTP *http.Client
@@ -0,0 +1,119 @@
package main
// The merge preview and the siblings of a fetch (novox/hq ADR 0484 clauses 3 and 4; issues 1675 and 1677). The merge
// itself is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes; what is here is the answer
// built from it and the placing of the sibling repositories the tests read.
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
)
// Sibling is a repository placed beside the fetched tree: its name (the directory under beside/) and its commit.
type Sibling struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Dir string `json:"dir"`
}
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
func (m *Module) remote(repository string) string {
base := m.RemoteBase
if base == "" {
base = "ssh://" + ForgeAlias + "/"
}
return base + repository + ".git"
}
// merge makes the merge preview of the head onto base and writes the answer into res.
func (m *Module) merge(ctx context.Context, res *FetchResult, tree, remote, base, commit string) error {
mr, err := buildstep.MergeInto(ctx, m.Run, tree, remote, base, commit, m.FetchOptions...)
if err != nil {
return err
}
res.Base = mr.Base
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
if res.Merge == "" {
return nil
}
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, nil, tree, res.Merge)
return err
}
type siblingArg struct{ repository, commit string }
func siblingsArg(v any) ([]siblingArg, error) {
if v == nil {
return nil, nil
}
list, ok := v.([]any)
if !ok {
return nil, fmt.Errorf("beside is a list of {repository, commit}")
}
var out []siblingArg
seen := map[string]bool{}
for _, e := range list {
o, _ := e.(map[string]any)
repository, _ := o["repository"].(string)
commit, _ := o["commit"].(string)
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("beside: %q is not a repository, <owner>/<name>", repository)
}
if !commitOK.MatchString(commit) {
return nil, fmt.Errorf("beside: %q is not a full commit (40 hex digits)", commit)
}
name := repository[strings.Index(repository, "/")+1:]
if seen[name] {
return nil, fmt.Errorf("beside: two repositories are named %s", name)
}
seen[name] = true
out = append(out, siblingArg{repository, commit})
}
return out, nil
}
// BesideDir is where a workspace's siblings are placed, and what MESH_CHECK_BESIDE names.
func (m *Module) besideDir(workspace string) string {
return filepath.Join(m.Work, workspace, "beside")
}
// placeSiblings fetches each sibling into beside/<name> of the workspace root, as a plain directory. A symlink anywhere
// in a placed tree fails the step naming it: the mesh creates none, and a test must not be led out of its sibling.
func (m *Module) placeSiblings(ctx context.Context, root string, siblings []siblingArg) ([]Sibling, error) {
var placed []Sibling
for _, s := range siblings {
name := s.repository[strings.Index(s.repository, "/")+1:]
dir := filepath.Join(root, "beside", name)
if err := os.MkdirAll(dir, 0o700); err != nil {
return nil, err
}
if err := buildstep.FetchCommit(ctx, m.Run, dir, m.remote(s.repository), s.commit, "", m.FetchOptions...); err != nil {
return nil, fmt.Errorf("beside %s: %w", s.repository, err)
}
if link := firstSymlink(dir); link != "" {
return nil, fmt.Errorf("beside %s holds a symlink, %s", s.repository, link)
}
placed = append(placed, Sibling{Repository: s.repository, Commit: s.commit, Dir: name})
}
sort.Slice(placed, func(i, j int) bool { return placed[i].Dir < placed[j].Dir })
return placed, nil
}
func firstSymlink(dir string) string {
var found string
filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err == nil && d.Type()&fs.ModeSymlink != 0 && found == "" {
found, _ = filepath.Rel(dir, p)
return fs.SkipAll
}
return nil
})
return found
}
@@ -0,0 +1,228 @@
//go:build integration
package main
// The fetch step's merge and siblings on real git repositories and the real go (issues 1675 and 1677): they start
// processes, so they carry the tag; preview_test.go proves the commands with a fake runner.
import (
"encoding/json"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func git(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
func put(t *testing.T, dir, name, body string) {
t.Helper()
p := filepath.Join(dir, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
// forge is a directory of bare repositories <owner>/<name>.git standing in for the forge, and a module fetching from it.
type forge struct {
t *testing.T
root string
}
func newForge(t *testing.T) *forge { return &forge{t: t, root: t.TempDir()} }
// repo makes the repository owner/name with the files on main and answers its work directory and the commit.
func (f *forge) repo(name string, files map[string]string) (work, commit string) {
bare := filepath.Join(f.root, "forge", name+".git")
work = filepath.Join(f.root, "work", name)
os.MkdirAll(bare, 0o755)
os.MkdirAll(work, 0o755)
git(f.t, bare, "init", "-q", "--bare")
git(f.t, work, "init", "-q", "-b", "main")
for n, c := range files {
put(f.t, work, n, c)
}
git(f.t, work, "add", ".")
git(f.t, work, "commit", "-q", "-m", "one")
git(f.t, work, "push", "-q", bare, "HEAD:refs/heads/main")
return work, git(f.t, work, "rev-parse", "HEAD")
}
// commit makes a commit off `from` in the repository's work directory and publishes it as the pull request's head.
func (f *forge) commit(name, work, from string, files map[string]string, ref string) string {
git(f.t, work, "checkout", "-q", "-B", "b"+ref[len(ref)-3:], from)
for n, c := range files {
put(f.t, work, n, c)
}
git(f.t, work, "add", ".")
git(f.t, work, "commit", "-q", "-m", "change")
sha := git(f.t, work, "rev-parse", "HEAD")
git(f.t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", name+".git"), "HEAD:"+ref)
return sha
}
func (f *forge) module(t *testing.T) (*Module, *keptStates) {
m, states := fetchModule(t, &fakeStep{})
m.Run = StepRunner(m.Home)
m.RemoteBase = "file://" + filepath.Join(f.root, "forge") + "/"
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
return m, states
}
// waitLong is waitEnd for a step that runs the real go, whose first build in a cold cache takes longer than 3 s.
func waitLong(t *testing.T, s *keptStates) delivery.Step {
t.Helper()
select {
case key := <-s.ended:
s.mu.Lock()
defer s.mu.Unlock()
v := s.steps[key]
return v[len(v)-1]
case <-time.After(2 * time.Minute):
t.Fatal("the step never ended on its key")
return delivery.Step{}
}
}
func fetched(t *testing.T, m *Module, states *keptStates, args map[string]any) (FetchResult, delivery.Step) {
t.Helper()
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
var res FetchResult
if st.State == delivery.StepPassed {
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
}
return res, st
}
var baseFiles = map[string]string{
"go.mod": "module fixture/m\n\ngo 1.26\n",
"a.txt": "one\ntwo\nthree\n",
"modules/m/module.json": "{\"name\":\"m\"}\n",
"modules/m/go.sum": "",
"modules/m/m.txt": "m\n",
"modules/other/other.txt": "o\n",
}
// 1675/AC1 and AC2 on real git: a clean merge answers the merge commit, the changed paths (the trunk's change is not
// among them) and the tree of each module directory, which equals git's own; a conflict is a passed step naming the files,
// leaving the workspace at the trunk commit; and no base is today's fetch.
func TestRealFetchMergesConflictsAndPlainlyFetches(t *testing.T) {
f := newForge(t)
work, fork := f.repo("o/r", baseFiles)
trunk := f.commit("o/r", work, fork, map[string]string{"modules/other/other.txt": "trunk\n", "a.txt": "one\ntwo\nTRUNK\n"}, "refs/heads/main")
clean := f.commit("o/r", work, fork, map[string]string{"modules/m/m.txt": "head\n"}, "refs/pull/1/head")
clash := f.commit("o/r", work, fork, map[string]string{"a.txt": "one\ntwo\nHEAD\n"}, "refs/pull/2/head")
m, states := f.module(t)
res, st := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
if st.State != delivery.StepPassed || res.Merge == "" || res.Base != trunk || res.Commit != clean || len(res.Conflicts) != 0 {
t.Fatalf("clean: %s %s", st.State, st.Result)
}
if strings.Join(res.Changed, ",") != "modules/m/m.txt" {
t.Errorf("changed = %v, want the head's own change alone", res.Changed)
}
tree := res.Trees["modules/m"]
if want := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", res.Merge+":modules/m"); tree.Tree != want {
t.Errorf("tree = %s, want %s", tree.Tree, want)
}
if tree.Manifest != sha256Hex([]byte("{\"name\":\"m\"}\n")) || tree.Locks["go.sum"] != sha256Hex(nil) {
t.Errorf("manifest and locks = %+v", tree)
}
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != res.Merge {
t.Errorf("the workspace is at %s, want the merge %s", got, res.Merge)
}
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
if st.State != delivery.StepPassed || res.Merge != "" || strings.Join(res.Conflicts, ",") != "a.txt" {
t.Fatalf("conflict: %s %s", st.State, st.Result)
}
if got := git(t, filepath.Join(m.Work, res.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
}
res, st = fetched(t, m, states, map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
if st.State != delivery.StepPassed || res.Merge != "" || res.Base != "" || res.Trees != nil || len(res.Modules) != 1 {
t.Errorf("no base: %s %s", st.State, st.Result)
}
}
// 1677/AC1 on the real go: the unit level of a repository whose test reads MESH_CHECK_BESIDE fails loudly without the
// sibling and passes with it; the sibling is a plain directory with no symlink in it; and a sibling that holds a
// symlink fails the fetch naming it.
func TestRealTestsReadTheSiblingBesideTheTree(t *testing.T) {
f := newForge(t)
files := map[string]string{}
for _, n := range []string{"go.mod", "beside.go", "beside_test.go"} {
raw, err := os.ReadFile(filepath.Join("testdata", "repos", "beside", n))
if err != nil {
t.Fatal(err)
}
files[n] = string(raw)
}
_, head := f.repo("o/r", files)
_, sibling := f.repo("o/sibling", map[string]string{"data.txt": "from the sibling\n"})
m, states := f.module(t)
without, _ := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head})
if _, err := m.testToolRun(t, without.Workspace); err != nil {
t.Fatal(err)
}
var res TestResult
st := waitLong(t, states)
json.Unmarshal(st.Result, &res)
if st.State != delivery.StepPassed || res.Passed || len(res.Failed) != 1 || !strings.Contains(res.Failed[0].Output, "MESH_CHECK_BESIDE is not set") {
t.Fatalf("without the sibling: %s %s", st.State, st.Result)
}
with, st2 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
"beside": []any{map[string]any{"repository": "o/sibling", "commit": sibling}}})
if st2.State != delivery.StepPassed || len(with.Beside) != 1 {
t.Fatalf("with the sibling: %s %s", st2.State, st2.Result)
}
m.testToolRun(t, with.Workspace)
res = TestResult{}
st = waitLong(t, states)
json.Unmarshal(st.Result, &res)
if !res.Passed {
t.Errorf("with the sibling the level failed: %s", st.Result)
}
if link := firstSymlink(filepath.Join(m.Work, with.Workspace, "beside")); link != "" {
t.Errorf("the placed tree holds a symlink, %s", link)
}
_, bad := f.repo("o/linked", map[string]string{"data.txt": "x\n"})
work := filepath.Join(f.root, "work", "o/linked")
if err := os.Symlink("/etc", filepath.Join(work, "etc-link")); err != nil {
t.Fatal(err)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "link")
bad = git(t, work, "rev-parse", "HEAD")
git(t, work, "push", "-q", "-f", filepath.Join(f.root, "forge", "o/linked.git"), "HEAD:refs/heads/main")
_, st3 := fetched(t, m, states, map[string]any{"repository": "o/r", "commit": head,
"beside": []any{map[string]any{"repository": "o/linked", "commit": bad}}})
if st3.State != delivery.StepFailed || !strings.Contains(st3.Reason, "etc-link") {
t.Errorf("a sibling with a symlink: %s %s", st3.State, st3.Reason)
}
}
@@ -0,0 +1,317 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const (
baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
mergeSHA = "fedcba9876543210fedcba9876543210fedcba98"
treeSHA = "1111111111111111111111111111111111111111"
)
// previewRules answers the merge's commands as git does: the merge commit, the changed paths, the manifests, the trees.
func previewRules(conflict bool) func(string) (forgekey.Ran, bool) {
merged := false
return func(line string) (forgekey.Ran, bool) {
switch {
case strings.Contains(line, "--is-shallow-repository"):
return forgekey.Ran{Stdout: "true\n"}, true
case strings.Contains(line, " merge --no-ff"):
if conflict {
return forgekey.Ran{Status: 1, Stderr: "CONFLICT (content): Merge conflict in a.go\n"}, true
}
merged = true
return forgekey.Ran{}, true
case strings.Contains(line, "diff --name-only --diff-filter=U"):
return forgekey.Ran{Stdout: "a.go\nsub/b.go\n"}, true
case strings.Contains(line, "diff --name-only --no-renames"):
return forgekey.Ran{Stdout: "a.go\nmodules/m/main.go\n"}, true
case strings.Contains(line, " rev-parse HEAD") && merged:
return forgekey.Ran{Stdout: mergeSHA + "\n"}, true
case strings.Contains(line, " ls-files ") && strings.Contains(line, "module.json"):
return forgekey.Ran{Stdout: "modules/m/module.json\n"}, true
case strings.Contains(line, " rev-parse "+mergeSHA+":modules/m"):
return forgekey.Ran{Stdout: treeSHA + "\n"}, true
}
return forgekey.Ran{}, false
}
}
// 1675/AC1: with a base the fetch merges the head onto it with the SDK's merge (the identity and flags of the check
// merge, hooks off), and answers base, merge commit, changed paths and, per module directory, tree, manifest hash and lock hashes.
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(false)}
m, states := fetchModule(t, f)
m.Run = func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran {
// the workspace's module.json and go.sum are on disk once the fake checks the head out
for i, a := range args {
if a == "-C" && len(args) > i+1 && strings.Contains(strings.Join(args, " "), " checkout ") {
os.MkdirAll(filepath.Join(args[i+1], "modules", "m"), 0o755)
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "module.json"), []byte("{}\n"), 0o644)
os.WriteFile(filepath.Join(args[i+1], "modules", "m", "go.sum"), []byte("sum\n"), 0o644)
}
}
return f.run(ctx, env, name, args...)
}
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
var res FetchResult
if err := json.Unmarshal(st.Result, &res); err != nil {
t.Fatal(err)
}
if res.Base != baseSHA || res.Merge != mergeSHA || res.Commit != commitSHA || len(res.Conflicts) != 0 ||
strings.Join(res.Changed, ",") != "a.go,modules/m/main.go" {
t.Errorf("result = %s", st.Result)
}
mt, ok := res.Trees["modules/m"]
if !ok || mt.Tree != treeSHA || mt.Manifest != sha256Hex([]byte("{}\n")) || mt.Locks["go.sum"] != sha256Hex([]byte("sum\n")) {
t.Errorf("trees = %+v", res.Trees)
}
lines := strings.Join(f.lines(), "\n")
for _, want := range []string{"fetch --no-tags --unshallow origin " + baseSHA, "checkout -q --detach " + baseSHA,
"core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
if !strings.Contains(lines, want) {
t.Errorf("git was not asked %q:\n%s", want, lines)
}
}
}
// 1675/AC2: a conflict is an answer: the step ends passed, the files are named, no merge commit, nothing downloaded.
func TestFetchWithAConflictEndsPassedAndNamesTheFiles(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n", rule: previewRules(true)}
m, states := fetchModule(t, f)
args := map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": baseSHA}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
if st.State != delivery.StepPassed {
t.Fatalf("a conflict ended the step %s: %s", st.State, st.Reason)
}
var res FetchResult
json.Unmarshal(st.Result, &res)
if res.Merge != "" || res.Base != baseSHA || strings.Join(res.Conflicts, ",") != "a.go,sub/b.go" || len(res.Changed) != 0 {
t.Errorf("result = %s", st.Result)
}
lines := strings.Join(f.lines(), "\n")
if !strings.Contains(lines, "merge --abort") || strings.Contains(lines, "mod download") {
t.Errorf("the merge was not undone, or modules were downloaded:\n%s", lines)
}
}
// 1675/AC3: no base is today's fetch: no merge, no extra command, and the answer carries none of the preview.
func TestFetchWithoutABaseIsTodaysFetch(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
fetchTool(t, m)(fetchArgs)
st := states.waitEnd(t)
lines := strings.Join(f.lines(), "\n")
if strings.Contains(lines, "merge") || strings.Contains(lines, "unshallow") {
t.Errorf("a fetch without a base merged:\n%s", lines)
}
for _, key := range []string{"base", "merge", "conflicts", "changed", "trees", "beside"} {
if strings.Contains(string(st.Result), `"`+key+`"`) {
t.Errorf("the answer carries %q without a base: %s", key, st.Result)
}
}
}
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "base": "main"})
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
t.Errorf("the step ended %s after %d commands", st.State, len(f.lines()))
}
}
// 1677/AC1: each sibling is fetched into beside/<name> of the workspace, and the tests of the workspace are given
// MESH_CHECK_BESIDE naming that directory, at the unit level; a workspace without siblings is given none.
func TestSiblingsArePlacedBesideAndTheUnitLevelIsGivenTheirDirectory(t *testing.T) {
f := &fakeStep{gomods: "go.mod\n"}
m, states := fetchModule(t, f)
args := map[string]any{"repository": "novox/mesh-controller", "commit": commitSHA, "beside": []any{
map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA},
map[string]any{"repository": "novox/mesh-host", "commit": commitSHA}}}
if _, err := fetchTool(t, m)(args); err != nil {
t.Fatal(err)
}
st := states.waitEnd(t)
var res FetchResult
json.Unmarshal(st.Result, &res)
if st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
if len(res.Beside) != 2 || res.Beside[0].Dir != "mesh-catalog" || res.Beside[1].Dir != "mesh-host" {
t.Fatalf("result = %s", st.Result)
}
lines := strings.Join(f.lines(), "\n")
for _, d := range []string{"beside/mesh-catalog", "beside/mesh-host"} {
if !strings.Contains(lines, filepath.Join(m.Work, res.Workspace, d)+" init") && !strings.Contains(lines, "-C "+filepath.Join(m.Work, res.Workspace, d)) {
t.Errorf("nothing was fetched into %s:\n%s", d, lines)
}
}
if !strings.Contains(lines, "remote add origin ssh://mesh-forge/novox/mesh-catalog.git") {
t.Errorf("the sibling was not fetched over ssh:\n%s", lines)
}
calls := &[]cmdRecord{}
m.Run = capturedRunner(t, "failing", calls)
if err := os.MkdirAll(filepath.Join(m.Work, res.Workspace, "tree"), 0o755); err != nil {
t.Fatal(err)
}
if _, err := m.testToolRun(t, res.Workspace); err != nil {
t.Fatal(err)
}
states.waitEnd(t)
want := "MESH_CHECK_BESIDE=" + filepath.Join(m.Work, res.Workspace, "beside")
if !envHas(*calls, "go", want) {
t.Errorf("no go command was given %s", want)
}
}
func (m *Module) testToolRun(t *testing.T, ws string) (any, error) {
t.Helper()
for _, tool := range m.Tools(m.StepTools()) {
if tool.Name == "build_go_test" {
return tool.Run(map[string]any{"workspace": ws, "level": "unit"})
}
}
t.Fatal("no tool build_go_test")
return nil, nil
}
func envHas(calls []cmdRecord, name, want string) bool {
for _, c := range calls {
if strings.HasPrefix(c.line, name+" ") {
for _, e := range c.env {
if e == want {
return true
}
}
}
}
return false
}
func TestAWorkspaceWithoutSiblingsIsGivenNoBesideVariable(t *testing.T) {
m, states, ws, calls := workspaceFor(t, "failing")
runUnit(t, m, states, ws)
for _, c := range *calls {
for _, e := range c.env {
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
t.Errorf("%s was given %s", c.line, e)
}
}
}
}
func TestSiblingsThatAreNotRepositoriesOrCommitsAreRefused(t *testing.T) {
for name, beside := range map[string]any{
"not a list": "x",
"bad repository": []any{map[string]any{"repository": "../x/y", "commit": baseSHA}},
"bad commit": []any{map[string]any{"repository": "novox/a", "commit": "main"}},
"twice": []any{map[string]any{"repository": "novox/a", "commit": baseSHA}, map[string]any{"repository": "other/a", "commit": baseSHA}},
} {
f := &fakeStep{}
m, states := fetchModule(t, f)
fetchTool(t, m)(map[string]any{"repository": "novox/mesh-sdk", "commit": commitSHA, "beside": beside})
if st := states.waitEnd(t); st.State != delivery.StepFailed || len(f.lines()) != 0 {
t.Errorf("%s: the step ended %s after %d commands", name, st.State, len(f.lines()))
}
}
}
// 1288/AC5 (the checker's contract): an import path runs only in the go.mod directory whose module holds it; a
// relative pattern runs in every directory; no pattern is ./... everywhere.
func TestPackagesRunOnlyInTheModuleThatHoldsThem(t *testing.T) {
tree := t.TempDir()
for dir, mod := range map[string]string{".": "example.org/root", "tools": "example.org/root/tools", "other": "example.org/other"} {
os.MkdirAll(filepath.Join(tree, dir), 0o755)
os.WriteFile(filepath.Join(tree, dir, "go.mod"), []byte("module "+mod+"\n\ngo 1.26\n"), 0o644)
}
dirs := []string{".", "other", "tools"}
got, err := packagesByDir(tree, dirs, []string{"example.org/root/a", "example.org/root/tools/b", "example.org/other"})
if err != nil {
t.Fatal(err)
}
want := map[string]string{".": "example.org/root/a", "tools": "example.org/root/tools/b", "other": "example.org/other"}
for d, w := range want {
if strings.Join(got[d], ",") != w {
t.Errorf("%s runs %v, want %s", d, got[d], w)
}
}
got, _ = packagesByDir(tree, dirs, []string{"./x/..."})
if len(got["."]) != 1 || len(got["tools"]) != 1 || len(got["other"]) != 1 {
t.Errorf("a relative pattern = %v", got)
}
got, _ = packagesByDir(tree, dirs, nil)
if got["tools"][0] != "./..." {
t.Errorf("no pattern = %v", got)
}
if _, err := packagesByDir(tree, dirs, []string{"example.org/nowhere/x"}); err == nil || !strings.Contains(err.Error(), "example.org/nowhere/x") {
t.Errorf("a package no module holds: %v", err)
}
}
func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) }
// 1677/AC (integration level): the integration level's go test is given MESH_CHECK_BESIDE when the workspace has siblings
// beside it, with the store's and the bus's addresses, and is given none when it has not.
func TestTheIntegrationLevelIsGivenTheBesideDirectoryOnlyWhenSiblingsWerePlaced(t *testing.T) {
for _, siblings := range []bool{true, false} {
f := newIntegFake(t, "slow")
m, states := integModule(t, f)
beside := filepath.Join(m.Work, "ws1", "beside")
if siblings {
if err := os.MkdirAll(filepath.Join(beside, "mesh-catalog"), 0o755); err != nil {
t.Fatal(err)
}
}
if _, st := runLevelTool(t, m, states, "integration"); st.State != delivery.StepPassed {
t.Fatalf("the step ended %s: %s", st.State, st.Reason)
}
tested := 0
for _, c := range f.calls {
if !strings.HasPrefix(c.line, "go ") || !strings.Contains(c.line, "-tags integration") {
continue
}
tested++
got := ""
hasNATS := false
for _, e := range c.env {
if strings.HasPrefix(e, "MESH_CHECK_BESIDE=") {
got = strings.TrimPrefix(e, "MESH_CHECK_BESIDE=")
}
hasNATS = hasNATS || strings.HasPrefix(e, "MESH_TEST_NATS=")
}
want := ""
if siblings {
want = beside
}
if got != want || !hasNATS {
t.Errorf("siblings %v: %s was given MESH_CHECK_BESIDE=%q (want %q), MESH_TEST_NATS present %v", siblings, c.line, got, want, hasNATS)
}
}
if tested == 0 {
t.Errorf("siblings %v: no integration go test ran", siblings)
}
}
}
@@ -20,7 +20,7 @@ const PostgresMajor = 18
// RecipeSteps is the recipe's own version: changed whenever the steps of fetch, test or build change in a way
// that can change what they produce (ADR 0405 §7). It is one of the three values the recipe version hashes.
const RecipeSteps = "1"
const RecipeSteps = "2"
// natsModule is the library the integration level's bus is started from (ADR 0470 §2).
const natsModule = "github.com/nats-io/nats-server/v2"
@@ -65,13 +65,23 @@ func (m *Module) test(ctx context.Context, args map[string]any) (any, error) {
}
switch level {
case LevelUnit:
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}})
return m.runLevel(ctx, tree, levelOpts{level: LevelUnit, packages: packages, bounds: Bounds{Run: m.unitBound()}, env: m.besideEnv(workspace)})
case LevelIntegration:
return m.integration(ctx, workspace, tree, packages)
}
return nil, fmt.Errorf("the level %q is neither unit nor integration", level)
}
// besideEnv is MESH_CHECK_BESIDE, naming the directory build_go_fetch placed the workspace's siblings in (ADR 0484
// clause 4), when it placed any; the tests that read other repositories look there (mesh-sdk go/internal/beside).
func (m *Module) besideEnv(workspace string) []string {
dir := m.besideDir(workspace)
if st, err := os.Stat(dir); err != nil || !st.IsDir() {
return nil
}
return []string{"MESH_CHECK_BESIDE=" + dir}
}
func packagesArg(v any) ([]string, error) {
if v == nil {
return nil, nil
@@ -115,12 +125,16 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
return nil, err
}
res := &TestResult{Level: o.level, Packages: []PackageResult{}, Setup: o.setup.Seconds(), Store: o.store, Bus: o.bus}
packages := o.packages
if len(packages) == 0 {
packages = []string{"./..."}
byDir, err := packagesByDir(tree, dirs, o.packages)
if err != nil {
return nil, err
}
began := m.clock()
for _, d := range dirs {
packages := byDir[d]
if len(packages) == 0 {
continue // no package asked for lies in this module
}
args := []string{"-C", filepath.Join(tree, d), "test", "-json", "-count=1", "-timeout", testTimeout}
if len(o.tags) > 0 {
args = append(args, "-tags", strings.Join(o.tags, ","))
@@ -141,3 +155,50 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR
res.judge(o.bounds)
return res, nil
}
// packagesByDir says which patterns run in which go.mod directory. A relative pattern (./...) runs in every directory, as
// it always did; an import path, as build_go_digest keys its packages, runs only in the directory whose module holds it
// (the longest module path that is the path or its prefix), as another module's path would fail the step. No pattern
// means ./... everywhere. An import path no module of the tree holds is an error that names it.
func packagesByDir(tree string, dirs, patterns []string) (map[string][]string, error) {
out := map[string][]string{}
if len(patterns) == 0 {
patterns = []string{"./..."}
}
modules := map[string]string{}
for _, d := range dirs {
if raw, err := os.ReadFile(filepath.Join(tree, d, "go.mod")); err == nil {
modules[d] = modulePath(string(raw))
}
}
for _, p := range patterns {
if strings.HasPrefix(p, "./") || p == "." {
for _, d := range dirs {
out[d] = append(out[d], p)
}
continue
}
owner, best := "", -1
for _, d := range dirs {
mp := modules[d]
if mp != "" && (p == mp || strings.HasPrefix(p, mp+"/")) && len(mp) > best {
owner, best = d, len(mp)
}
}
if best < 0 {
return nil, fmt.Errorf("no module of the workspace holds the package %s", p)
}
out[owner] = append(out[owner], p)
}
return out, nil
}
// modulePath is the module path a go.mod declares, "" when it declares none.
func modulePath(gomod string) string {
for _, l := range strings.Split(gomod, "\n") {
if f := strings.Fields(l); len(f) == 2 && f[0] == "module" {
return strings.Trim(f[1], `"`)
}
}
return ""
}
@@ -0,0 +1 @@
package beside
@@ -0,0 +1,22 @@
package beside
import (
"os"
"path/filepath"
"testing"
)
// A test of a repository that reads another one, as the controller's, the node-engine's, the SDK's and the catalogue's do.
func TestReadsTheSiblingBesideIt(t *testing.T) {
root := os.Getenv("MESH_CHECK_BESIDE")
if root == "" {
t.Fatal("the sibling is not beside this check: MESH_CHECK_BESIDE is not set")
}
raw, err := os.ReadFile(filepath.Join(root, "sibling", "data.txt"))
if err != nil {
t.Fatal(err)
}
if string(raw) != "from the sibling\n" {
t.Fatalf("data.txt = %q", raw)
}
}
@@ -0,0 +1,3 @@
module fixture/beside
go 1.26
+1 -1
View File
@@ -3,7 +3,7 @@ module meshbuildgo
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
)
+4
View File
@@ -1,5 +1,9 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -36,6 +36,8 @@ const sdkPackage = "@novox/mesh-sdk"
// WorkspaceAge is how long a workspace is kept.
const WorkspaceAge = 24 * time.Hour
var commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`)
var repositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// FetchResult is what a passed fetch step says.
@@ -48,6 +50,15 @@ type FetchResult struct {
// Refused are the bundles whose packages were not installed, each with why (a module naming packages beyond the SDK
// with no package-lock.json). The fetch passes; a build of such a bundle fails naming it (issue 1459).
Refused []Refusal `json:"refused"`
// The merge preview (ADR 0484 clause 3), only when the call gave a base, as build_go_fetch answers it: the trunk
// commit, the merge commit of the head onto it, the paths the merge changes and the tree of each module directory.
// A head that does not merge cleanly is an answer: Merge is empty, Conflicts names the files, and the workspace is
// left at the trunk commit with nothing installed.
Base string `json:"base,omitempty"`
Merge string `json:"merge,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]buildstep.ModuleTree `json:"trees,omitempty"`
}
// Refusal is a module a fetch did not install the packages of.
@@ -68,6 +79,7 @@ func (m *Module) fetchTool() StepTool {
"repository": map[string]any{"type": "string", "description": "<owner>/<repository> on the forge"},
"commit": map[string]any{"type": "string", "description": "the full commit, 40 hex digits"},
"ref": map[string]any{"type": "string", "description": "refs/pull/<n>/head when only a pull request's head carries the commit"},
"base": map[string]any{"type": "string", "description": "optional trunk commit, 40 hex digits: the head is merged onto it as the forge's merge would, and the answer carries the merge commit, the files in conflict (an answer, the workspace is then left at the trunk commit), the changed paths and the tree of each module directory"},
},
"required": []string{"repository", "commit"},
},
@@ -80,6 +92,10 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
commit, _ := args["commit"].(string)
ref, _ := args["ref"].(string)
base, _ := args["base"].(string)
if base != "" && !commitOK.MatchString(base) {
return nil, fmt.Errorf("%q is not a trunk commit (40 hex digits)", base)
}
if !repositoryOK.MatchString(repository) || strings.Contains(repository, "..") {
return nil, fmt.Errorf("%q is not a repository, <owner>/<name>", repository)
}
@@ -94,15 +110,23 @@ func (m *Module) fetch(ctx context.Context, args map[string]any) (any, error) {
gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran {
return m.Run(ctx, append(m.baseEnv(), env...), name, a...)
}
remote := "ssh://" + ForgeAlias + "/" + repository + ".git"
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref); err != nil {
remote := m.remote(repository)
if err := buildstep.FetchCommit(ctx, gitRun, tree, remote, commit, ref, m.FetchOptions...); err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
if base != "" {
if err := m.merge(ctx, gitRun, &res, tree, remote, base, commit); err != nil {
return nil, err
}
if res.Merge == "" {
return res, nil // a conflict: the tree is at the trunk, nothing of the head is installed
}
}
dirs, err := m.typeScriptModules(ctx, tree)
if err != nil {
return nil, err
}
res := FetchResult{Workspace: id, Commit: commit, Modules: []string{}, Installed: []string{}, Refused: []Refusal{}}
for _, d := range dirs {
res.Modules = append(res.Modules, d)
did, refused, err := m.installOwn(ctx, root, filepath.Join(tree, filepath.FromSlash(d)), d)
@@ -48,7 +48,7 @@ func (f *fakeMachine) run(_ context.Context, env []string, name string, args ...
}
} else if name == "git" {
for _, a := range args {
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files"}, a) {
if slices.Contains([]string{"init", "remote", "fetch", "checkout", "rev-parse", "ls-files", "merge", "diff"}, a) {
key = "git " + a
break
}
@@ -31,9 +31,13 @@ type Module struct {
Toolchain *Toolchain
// Work is the work directory, outside the home: a workspace is <Work>/<step id>/tree. Run runs the commands of a
// step (a fake in a test). Clock is the time the steps are timed by.
Work string
Run forgekey.Runner
Clock func() time.Time
Work string
Run forgekey.Runner
// RemoteBase is where repositories are fetched from, "ssh://mesh-forge/" when empty; FetchOptions are the options of
// the SDK's fetch and merge. A test points them at a local repository (buildstep.AllowFileRemote).
RemoteBase string
FetchOptions []buildstep.Option
Clock func() time.Time
// Toolchains and Steps are the Put of the states of those names.
Toolchains, Steps buildstep.Put
// ReadSteps and ReadToolchainsState read those two states back for the read tools (issue 1562).
@@ -0,0 +1,35 @@
package main
// The merge preview of a fetch (novox/hq ADR 0484 clause 3; issue 1676), as build_go_fetch answers it. The merge itself
// is the SDK's (buildstep.MergeInto), the one every build module's fetch step makes.
import (
"context"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
// remote is the address a repository is fetched from: the forge over ssh as the account, or RemoteBase when a test set it.
func (m *Module) remote(repository string) string {
base := m.RemoteBase
if base == "" {
base = "ssh://" + ForgeAlias + "/"
}
return base + repository + ".git"
}
// merge makes the merge preview of the head onto base and writes the answer into res.
func (m *Module) merge(ctx context.Context, run forgekey.Runner, res *FetchResult, tree, remote, base, commit string) error {
mr, err := buildstep.MergeInto(ctx, run, tree, remote, base, commit, m.FetchOptions...)
if err != nil {
return err
}
res.Base = mr.Base
res.Conflicts, res.Merge, res.Changed = mr.Conflicts, mr.Merge, mr.Changed
if res.Merge == "" {
return nil
}
res.Trees, err = buildstep.ModuleTrees(ctx, m.Run, m.baseEnv(), tree, res.Merge)
return err
}
@@ -0,0 +1,95 @@
//go:build integration
package main
// The fetch step's merge on real git repositories (issue 1676); preview_test.go proves the commands with a fake runner.
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
func git(t *testing.T, dir string, args ...string) string {
t.Helper()
cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.name=t", "-c", "user.email=t@example.invalid", "-c", "commit.gpgsign=false"}, args...)...)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v: %v\n%s", args, err, out)
}
return strings.TrimSpace(string(out))
}
func TestRealFetchMergesAConflictAndAPlainFetch(t *testing.T) {
root := t.TempDir()
bare, work := filepath.Join(root, "forge", "o", "r.git"), filepath.Join(root, "work")
os.MkdirAll(bare, 0o755)
os.MkdirAll(work, 0o755)
git(t, bare, "init", "-q", "--bare")
git(t, work, "init", "-q", "-b", "main")
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`)
files["a.txt"] = "one\ntwo\nthree\n"
for n, c := range files {
write(t, filepath.Join(work, n), c, 0o644)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "fork")
fork := git(t, work, "rev-parse", "HEAD")
change := func(from, ref string, f map[string]string) string {
git(t, work, "checkout", "-q", "-B", "tmp", from)
for n, c := range f {
write(t, filepath.Join(work, n), c, 0o644)
}
git(t, work, "add", ".")
git(t, work, "commit", "-q", "-m", "c")
git(t, work, "push", "-q", "-f", bare, "HEAD:"+ref)
return git(t, work, "rev-parse", "HEAD")
}
trunk := change(fork, "refs/heads/main", map[string]string{"a.txt": "one\ntwo\nTRUNK\n"})
clean := change(fork, "refs/pull/1/head", map[string]string{"modules/m/extra.ts": "export {}\n"})
clash := change(fork, "refs/pull/2/head", map[string]string{"a.txt": "one\ntwo\nHEAD\n"})
m, _, _ := newMachine(t)
m.Run = forgekey.ExecRunner
m.RemoteBase = "file://" + filepath.Join(root, "forge") + "/"
m.FetchOptions = []buildstep.Option{buildstep.AllowFileRemote()}
out, err := m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head", "base": trunk})
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
tree := filepath.Join(m.Work, r.Workspace, "tree")
if r.Merge == "" || strings.Join(r.Changed, ",") != "modules/m/extra.ts" || len(r.Conflicts) != 0 || len(r.Modules) != 1 {
t.Fatalf("clean: %+v", r)
}
if want := git(t, tree, "rev-parse", r.Merge+":modules/m"); r.Trees["modules/m"].Tree != want {
t.Errorf("tree = %s, want %s", r.Trees["modules/m"].Tree, want)
}
if got := git(t, tree, "rev-parse", "HEAD"); got != r.Merge {
t.Errorf("the workspace is at %s, want the merge %s", got, r.Merge)
}
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clash, "ref": "refs/pull/2/head", "base": trunk})
if err != nil {
t.Fatalf("a conflict failed the step: %v", err)
}
r = out.(FetchResult)
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.txt" {
t.Fatalf("conflict: %+v", r)
}
if got := git(t, filepath.Join(m.Work, r.Workspace, "tree"), "rev-parse", "HEAD"); got != trunk {
t.Errorf("after a conflict the workspace is at %s, want the trunk %s", got, trunk)
}
out, err = m.fetch(context.Background(), map[string]any{"repository": "o/r", "commit": clean, "ref": "refs/pull/1/head"})
if err != nil || out.(FetchResult).Merge != "" || out.(FetchResult).Trees != nil {
t.Errorf("no base: %+v %v", out, err)
}
}
@@ -0,0 +1,87 @@
package main
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/forgekey"
)
const baseSHA = "abcdef0123456789abcdef0123456789abcdef01"
func commandLines(f *fakeMachine) string {
var out []string
for _, c := range f.calls {
out = append(out, joined(c))
}
return strings.Join(out, "\n")
}
// 1676/AC1: with a base the fetch merges the head onto it through the SDK's merge (the check merge's identity and flags,
// hooks off), and answers the base, the merge commit, the changed paths and the tree, manifest and locks of each module
// directory; the TypeScript modules are installed from the merged tree.
func TestFetchWithABaseMergesAndAnswersThePreview(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
f.answer["git diff"] = forgekey.Ran{Stdout: "modules/m/package.json\n"}
args := map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA}
out, err := m.fetch(context.Background(), args)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if r.Base != baseSHA || r.Merge != commitSHA || strings.Join(r.Changed, ",") != "modules/m/package.json" || len(r.Conflicts) != 0 {
t.Errorf("result = %+v", r)
}
if mt, ok := r.Trees["modules/m"]; !ok || mt.Tree != commitSHA || mt.Manifest == "" {
t.Errorf("trees = %+v", r.Trees)
}
if len(r.Modules) != 1 || r.Modules[0] != "modules/m" {
t.Errorf("modules = %v", r.Modules)
}
for _, want := range []string{"fetch --no-tags origin " + baseSHA, "core.hooksPath=/dev/null", "merge --no-ff --no-edit --no-verify"} {
if !strings.Contains(commandLines(f), want) {
t.Errorf("git was not asked %q:\n%s", want, commandLines(f))
}
}
}
// 1676/AC2: a conflict is an answer: the step passes with the files named, no merge commit, and nothing installed.
func TestFetchWithAConflictAnswersTheFilesAndInstallsNothing(t *testing.T) {
files := typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6","lodash":"^4"}}`)
files["modules/m/package-lock.json"] = `{"lockfileVersion":3}`
m, f := fetchMachine(t, files, nil)
f.answer["git merge"] = forgekey.Ran{Status: 1, Stderr: "CONFLICT\n"}
f.answer["git diff"] = forgekey.Ran{Stdout: "a.ts\nsub/b.ts\n"}
out, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": baseSHA})
if err != nil {
t.Fatalf("a conflict failed the step: %v", err)
}
r := out.(FetchResult)
if r.Merge != "" || strings.Join(r.Conflicts, ",") != "a.ts,sub/b.ts" || len(r.Changed) != 0 || len(npmCalls(f)) != 0 {
t.Errorf("result = %+v, npm calls %d", r, len(npmCalls(f)))
}
if !strings.Contains(commandLines(f), "merge --abort") {
t.Errorf("the merge was not undone:\n%s", commandLines(f))
}
}
// 1676/AC3: no base is today's fetch: no merge command, and none of the preview in the answer.
func TestFetchWithoutABaseMakesNoMerge(t *testing.T) {
m, f := fetchMachine(t, typeScriptModule(`{"dependencies":{"@novox/mesh-sdk":"^0.1.6"}}`), nil)
out, err := m.fetch(context.Background(), fetchArgs)
if err != nil {
t.Fatal(err)
}
r := out.(FetchResult)
if strings.Contains(commandLines(f), "merge") || strings.Contains(commandLines(f), "unshallow") || r.Base != "" || r.Merge != "" || r.Trees != nil {
t.Errorf("a fetch without a base merged: %+v\n%s", r, commandLines(f))
}
}
func TestABaseThatIsNotACommitIsRefusedBeforeAnythingRuns(t *testing.T) {
m, f := fetchMachine(t, nil, nil)
if _, err := m.fetch(context.Background(), map[string]any{"repository": "novox/mesh-catalog", "commit": commitSHA, "base": "main"}); err == nil || len(f.calls) != 0 {
t.Errorf("err = %v after %d commands", err, len(f.calls))
}
}
+1 -1
View File
@@ -2,4 +2,4 @@ module meshbuildnode
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
+4
View File
@@ -1,2 +1,6 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3 h1:OKxU64kHN1yQ/GOa6LwoUwVmiQ74d83WMSZJRFPVm9c=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021093645-465beb4ea5e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5 h1:1BiPKRPTYpgex8rSHWT9az0BteOafGWRDzd/Ba6GAcA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021185947-afbe371003a5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
@@ -380,18 +380,19 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) {
// A group is not asked while a member's required status is not a success at its head (issue 1275): the group
// says which, and is asked once every member may merge.
var holdMember, holdSays string
var holdOwn bool
var readErr error
for _, m := range reads {
if gid == "" {
break
}
says, _, err := h.notASuccess(m)
says, ownOnly, err := h.notASuccess(m)
if err != nil {
readErr = err
break
}
if says != "" {
holdMember, holdSays = m.id, says
holdMember, holdSays, holdOwn = m.id, says, ownOnly
break
}
}
@@ -421,7 +422,18 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) {
if readErr != nil || holdSays != "" {
stale()
h.mu.Lock()
defer h.mu.Unlock()
var own func()
// A member's own repository check pending or missing (a sibling's new head put it out of date, issue 1792) is
// asked again here, once for the head: nothing else asks it, and the composed check sets no repository check.
if d := h.deliveries[holdMember]; holdOwn && d != nil && d.Ready != nil {
own = h.askOwnAfterRefusal(d)
}
defer func() {
h.mu.Unlock()
if own != nil {
own()
}
}()
undo()
if g := h.groups[gid]; g != nil && holdSays != "" {
if g.Hold == nil || g.Hold.Kind != "unasked" || g.Hold.Says != holdSays || g.Hold.Member != holdMember {
@@ -634,6 +646,7 @@ func (h *Holder) answered(key string, ans sdk.Answer) []func() {
for _, d := range covered {
r := d.Ready
r.Warrant = ans.Words
r.Repeats = 0 // the operator chose: whatever the tool refused before is asked afresh (issue 1106)
h.mergeAgainNow(d) // the operator chose: the merge is asked at once
switch ans.Option {
case OptionApprove:
@@ -519,30 +519,29 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB
b.publish(t, "mesh.mod.gitea.event.pull.updated", announced)
passed := b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 7, head, "pass", "t1t1t1t1t1t1"))
// The ask is written into the gate's bucket asks within seconds of the verdict's event, with no label, comment or mark
// written by anyone, and nothing is asked of a router.
// No review names the head, so nothing is asked (ADR 0496, issue 1803): the delivery waits for a review.
rt.until("the delivery to wait for a review", 5*time.Second, func() bool {
e, err := b.kvs["pulls"].Get(ctx, "novox.app.7")
var k PullKey
return err == nil && json.Unmarshal(e.Value(), &k) == nil && k.Verdict == "pass"
})
time.Sleep(time.Second)
if key := onlyAsk(b); key != "" {
t.Fatalf("a head with no review was asked: %s", key)
}
// The description is edited to add the review record (issue 1079): the forge module's edit event carries it, and the
// ask is written into the gate's bucket asks within seconds, with no label, comment or mark written by anyone, and
// nothing is asked of a router.
edit := time.Now()
b.publish(t, "mesh.mod.gitea.event.forge.pull-request.edited", map[string]any{"owner": "novox", "repo": "app", "number": 7,
"title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head})
_ = passed
var key string
rt.until("the ask to be written", 5*time.Second, func() bool { key = onlyAsk(b); return key != "" })
ask := askAt(t, b, key)
t.Logf("the ask was written %s after the verdict's event", time.Since(passed))
if !strings.Contains(ask.Ask.Explanation, "not reviewed at this head") || ask.Ask.ID != sdk.AskID(key) {
t.Fatalf("the first ask does not say the record is missing: %s", ask.Ask.Explanation)
}
// The description is edited to add the review record (issue 1079): the forge module's edit event carries it. The unanswered
// ask is taken away and written again, once, with the record read (the heads are the same, so is the key).
b.publish(t, "mesh.mod.gitea.event.forge.pull-request.edited", map[string]any{"owner": "novox", "repo": "app", "number": 7,
"title": announced.Title, "head_sha": head, "base": "main", "body": "Reviewed-by: lead@shanks at " + head})
rt.until("the ask to be written again with the record", 5*time.Second, func() bool {
a, ok := askNow(b, key) // the key may be absent for an instant while the ask is taken away and written again
return ok && strings.Contains(a.Ask.Explanation, "reviewed by lead on shanks")
})
// The ask is overwritten, never taken away: the key is there at every revision, and the second asking has an id of its own
// (review round 1, G2), which the router would take as new.
if h, err := b.kvs["asks"].History(ctx, key); err != nil || len(h) != 2 || h[1].Operation() != jetstream.KeyValuePut {
t.Fatalf("the ask's history is %d revisions (%v); wanted written and written again", len(h), err)
}
if again := askAt(t, b, key); again.Asking != 2 || again.Ask.ID != sdk.AskingID(key, 2) || again.Ask.ID == ask.Ask.ID {
t.Fatalf("the second asking is %d under %s, the first was %s", again.Asking, again.Ask.ID, ask.Ask.ID)
t.Logf("the ask was written %s after the edit's event", time.Since(edit))
if !strings.Contains(ask.Ask.Explanation, "reviewed by lead on shanks") || ask.Ask.ID != sdk.AskID(key) {
t.Fatalf("the ask does not say the record: %s", ask.Ask.Explanation)
}
if len(rt.asksOf("gitea.gitea_merge_pull_request")) != 0 {
t.Fatalf("merged before the Approve")
@@ -190,7 +190,7 @@ func (h *Holder) deliveryOf(repository string, number int, commit string) *check
c.Check = &v
}
if g := h.groups[found.Group]; g != nil {
state, why := GroupState(g, h.membersOf(g))
state, why, _ := h.groupState(g, h.membersOf(g))
c.groupSaid = state + ": " + why
if g.Check != nil && g.Check.Verdict != "" {
c.groupSaid += "; composed check " + g.Check.Verdict + " — " + g.Check.Summary
@@ -131,6 +131,9 @@ func statusOf(d *Delivery) (string, string) {
case Proposed, Checked:
return "pending", "checking"
case Ready:
if d.holdsForReview() {
return "pending", clip("ready, "+d.Ready.Waits, 140) // novox/hq ADR 0496: no Approve is asked before a review
}
return "success", "ready: it delivers once merged"
case Rejected:
return "failure", clip("rejected: "+why, 140)
@@ -154,6 +157,9 @@ func statusOf(d *Delivery) (string, string) {
}
return "failure", clip("failed: "+why, 140)
case Superseded:
if d.coveredStop() {
return "success", clip(d.coveredStopSaid(), 140) // its walk's taker delivered it (issue 1786)
}
return "warning", clip("superseded: "+why, 140)
case Stopped:
if d.coveredStop() {
@@ -158,11 +158,11 @@ func (*askedAgainError) Error() string {
return "the forge refused gitea_merge_pull_request: novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. The check of head cccccccccccc merged into trunk bbbbbbbbbbbb was asked; took 3s. Ask again at cccccccccccc once both statuses pass. Nothing was merged"
}
// Issue 1022 (ADR 0421 §5): the review record is read and put in the ask; it is never a gate.
func TestTheAskSaysFirstWhenThePullRequestIsNotReviewedAtThisHead(t *testing.T) {
for _, c := range []struct{ name, body, want string }{
{"no line", "Fixes issue 7.", "not reviewed at this head"},
{"another head", "Reviewed-by: lead@shanks at " + pushed, "the Reviewed-by line names " + pushed[:12]},
// Issue 1022 (ADR 0421 §5), as ADR 0496 changed it (issue 1803): the review record is read; a head not reviewed is not asked.
func TestAHeadNotReviewedAtThisHeadIsNotAsked(t *testing.T) {
for _, c := range []struct{ name, body string }{
{"no line", "Fixes issue 7."},
{"another head", "Reviewed-by: lead@shanks at " + pushed},
} {
t.Run(c.name, func(t *testing.T) {
w := newWorld(t)
@@ -171,13 +171,8 @@ func TestTheAskSaysFirstWhenThePullRequestIsNotReviewedAtThisHead(t *testing.T)
p.Body = c.body
w.h.PullUpdated(p)
hear(t, w, ControllerSeat+".checked", judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
if w.op.count() != 1 {
t.Fatalf("an unreviewed pull request must still be asked: %d", w.op.count())
}
a := w.op.ask(w.op.open()[0])
first := strings.SplitN(a.Explanation, "\n", 2)[0]
if !strings.Contains(first, "not reviewed at this head") || !strings.Contains(a.Explanation, c.want) {
t.Fatalf("the ask does not say first that it is not reviewed at this head:\n%s", a.Explanation)
if w.op.count() != 0 {
t.Fatalf("an unreviewed pull request was asked: %d", w.op.count())
}
})
}
@@ -601,10 +601,11 @@ type world struct {
source *fakeSource
owed *fakeOwed
// asked is the key of the ask a test answered.
asked string
now time.Time
said []map[string]any
mu sync.Mutex
asked string
now time.Time
logbuf strings.Builder
said []map[string]any
mu sync.Mutex
}
func newWorld(t *testing.T) *world {
@@ -615,7 +616,7 @@ func newWorldOver(t *testing.T, store *memStore, ctl *fakeController, forge *fak
w := &world{t: t, store: store, ctl: ctl, forge: forge, now: now}
w.h = &Holder{Store: store, Controller: ctl, Forge: forge,
Now: func() time.Time { w.mu.Lock(); defer w.mu.Unlock(); return w.now },
Logf: func(format string, a ...any) { t.Logf(format, a...) },
Logf: func(format string, a ...any) { t.Logf(format, a...); fmt.Fprintf(&w.logbuf, format+"\n", a...) },
Emit: func(event string, body any) error {
w.mu.Lock()
defer w.mu.Unlock()
@@ -682,7 +683,9 @@ func (w *world) state(id string) State { return w.delivery(id).State }
func pr(repo string, number int, sha, branch string, paths ...string) PullEvent {
owner, name, _ := strings.Cut(repo, "/")
return PullEvent{Owner: owner, Repo: name, Number: number, Base: "main", Head: branch, HeadSHA: sha,
HTMLURL: fmt.Sprintf("https://forge.invalid/%s/pulls/%d", repo, number), Paths: paths}
HTMLURL: fmt.Sprintf("https://forge.invalid/%s/pulls/%d", repo, number), Paths: paths,
// Reviewed at its head, as the Approve is asked only for a reviewed head (novox/hq ADR 0496, issue 1803).
Body: "Reviewed-by: tester@shanks at " + sha}
}
// verdict is the controller's `checked` for a head.
@@ -719,3 +722,5 @@ func (f *fakeForge) Comment(owner, repo string, number int, body string) error {
f.calls = append(f.calls, "comment "+body)
return nil
}
func (w *world) logs() string { return w.logbuf.String() }
@@ -467,7 +467,7 @@ func (h *Holder) fillFromAnnouncement(d *Delivery, p PullEvent) {
// PullEdited takes a pull request's edit (novox/hq issue 1079): its description is read again for the review record, the
// `after:` lines and the issues it names, exactly as an announcement reads them. An Approve ask still unanswered whose words
// the new record changes is withdrawn and asked again with it read, without a word on the pull request: nothing was
// approved, so no Approve ended. An answered ask is left as it is. The record is never a gate (ADR 0421 §5). An edit with no
// approved, so no Approve ended. An answered ask is left as it is. A record naming the head lifts the hold at once (ADR 0496 §4). An edit with no
// description, or of a pull request with no open delivery at that head, changes nothing.
func (h *Holder) PullEdited(p PullEvent) {
if p.HeadSHA == "" || p.Number == 0 {
@@ -995,6 +995,12 @@ func (h *Holder) Checked(c CheckedEvent) {
}
h.moved(d, []Transition{t})
}
if d.Ready != nil && !d.Ready.OwnAskedAt.IsZero() {
// The own check asked for this head answered (issue 1792): one own ask is in flight at a time, so a status still
// pending or missing after its answer is asked again.
d.Ready.OwnAskedAt = time.Time{}
h.keep(d)
}
if d.State == Rejected && c.Trunk != "" && d.Check != nil && !sameCommit(d.Check.Trunk, c.Trunk) {
// Judged again against another trunk commit — the merge tool's ask of a verdict out of date, or a check asked
// elsewhere (novox/hq ADR 0323 §5, §7): taken as the recheck it is.
@@ -1390,10 +1396,31 @@ func (h *Holder) plan() []func() {
// Following a walk that was taken over: the walk that took it over, read as soon as it is heard of, asked for
// by its id when it is not.
for _, d := range h.deliveries {
if d.State.Final() || d.Walk == nil || d.Walk.TakenOverBy == "" {
if d.Walk == nil || d.Walk.TakenOverBy == "" {
continue
}
if d.State.Final() {
// Ended superseded by its walk before the walk that took it over was heard to answer (issue 1786): the
// state stays final, as a covered superseded member's does (issue 442), and the end of that walk is
// kept as DeliveredBy, which the group and the views read. Never logged, never changed otherwise.
if d.supersededByItsWalk() && d.DeliveredBy == "" {
if s, known := h.walks[d.Walk.TakenOverBy]; known {
if _, ok := walkAnswers(s.walk, d); ok && s.walk.State == walkDone {
d.DeliveredBy = fmt.Sprintf("the walk %s, which took its walk %s over, ended done", s.walk.ID, d.Walk.ID)
h.tellCovered(d, now)
}
} else if a := h.askWalk(d.Walk.TakenOverBy, "which took a delivery's walk over", now); a != nil {
asks = append(asks, a)
}
}
continue
}
if s, known := h.walks[d.Walk.TakenOverBy]; known {
if s.walk.IsBatch() {
// A batch is no walk yet (ADR 0276): the walk it is cut into answers the merge. Waited for, never
// read as a walk that does not answer it (issue 1786).
continue
}
if c, ok := walkAnswers(s.walk, d); ok {
h.linkWalk(d, s.walk, c)
} else {
@@ -2175,7 +2202,7 @@ func (h *Holder) planGroup(g *Group, members []*Delivery, now time.Time) []func(
// moves — and its group counts it as reached (novox/hq issue 442).
if g.Closed {
for _, d := range members {
if d.State != Superseded || d.CoveredBy != "" {
if d.State != Superseded || d.CoveredBy != "" || d.DeliveredBy != "" {
continue
}
var ask []string
@@ -2190,15 +2217,22 @@ func (h *Holder) planGroup(g *Group, members []*Delivery, now time.Time) []func(
}
}
// Its state, said when it changes.
state, why := GroupState(g, members)
state, why, held := h.groupState(g, members)
status := groupStatusState(state)
if held {
status = "pending" // no Approve exists while a member waits for its review (novox/hq ADR 0496 §2)
}
if state == "delivering" && !(g.Check != nil && (g.Check.Verdict == "pass" || g.Check.Verdict == "warning") &&
sameMembers(g.Check.Commits, commitsOf(members)) && g.Check.OutOfDate == "") {
// Success there lets the merge tool merge its members: only for a composed check that passed for these heads.
status = "pending"
}
if state != g.Said || status != g.SaidStatus {
g.Said, g.SaidStatus = state, status
reviewSaid := ""
if held {
reviewSaid = why
}
if state != g.Said || status != g.SaidStatus || reviewSaid != g.SaidReview {
g.Said, g.SaidStatus, g.SaidReview = state, status, reviewSaid
for _, d := range members {
g.Owed = append(g.Owed, Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery-group",
State: status, Line: clip(fmt.Sprintf("group %s %s: %s", g.ID, state, why), 140),
@@ -2692,6 +2726,29 @@ func GroupState(g *Group, members []*Delivery) (string, string) {
return "checking", fmt.Sprintf("%d of %d member(s) ready", count[Ready], n)
}
// groupState is GroupState, but a group whose members all passed and compose waits for a review while any member the
// Approve would cover has none at its head: its why names the members (novox/hq ADR 0496 §2, issue 1803), and held
// says its mesh/delivery-group is pending, not a success. Under the lock.
func (h *Holder) groupState(g *Group, members []*Delivery) (state, why string, held bool) {
state, why = GroupState(g, members)
if state != "ready" || g.Closed {
return state, why, false
}
var waiting []string
for _, m := range members {
if m.MergedAs != "" || m.State.Final() || h.covering(m) != nil {
continue
}
if _, ok := reviewWords(m); !ok {
waiting = append(waiting, "at "+shortOf(m.Commit, 12)+" ("+pullSaid(m)+")")
}
}
if len(waiting) == 0 {
return state, why, false
}
return state, waitsForReview + strings.Join(waiting, " and ")[len("at "):], true
}
// supersededWhy begins why a group whose members all ended is stopped. It is kept short: the forge's status line,
// "group <id> stopped: <why>", is clipped at 140 characters, and the member's id must fit in it whole.
const supersededWhy = "superseded, not carried by a later delivery: "
@@ -19,8 +19,12 @@ const (
func TestAPullRequestsHeadIsProposedCheckedAndReady(t *testing.T) {
w := newWorld(t)
w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go"))
p := pr("novox/app", 7, head, "feat/x", "src/a.go")
p.Body = "" // an empty description is never read again, so the review set below stays
w.h.PullUpdated(p)
id := IDOf("novox/app", head)
// This head is not a full commit id, so its description line cannot name it: its review is set as read.
w.h.deliveries[id].ReviewedBy, w.h.deliveries[id].ReviewedAt = "tester@shanks", head
if w.state(id) != Proposed {
t.Fatalf("a head is %s", w.state(id))
}
@@ -359,6 +363,9 @@ func TestAGroupIsCheckedTogetherAndLetGoOnceEveryMemberIsMerged(t *testing.T) {
ctl, cat := IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead)
w.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"))
w.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json"))
// These heads are not full commit ids, so their description lines cannot name them: the reviews are set as read.
w.h.deliveries[ctl].ReviewedBy, w.h.deliveries[ctl].ReviewedAt = "tester@shanks", ctlHead
w.h.deliveries[cat].ReviewedBy, w.h.deliveries[cat].ReviewedAt = "tester@shanks", catHead
if w.delivery(ctl).Group != "feat/x" || w.delivery(cat).Group != "feat/x" {
t.Fatalf("one branch name in two repositories made no group: %q %q", w.delivery(ctl).Group, w.delivery(cat).Group)
}
@@ -10,18 +10,19 @@ import (
// this head". The forge's edit event is followed; an Approve ask not yet answered is withdrawn and asked again with the
// record read (the operator agreed, 2026-10-18). The review record is never a gate (ADR 0421 §5).
// askedUnreviewed is a green pull request whose Approve is asked while its description holds no Reviewed-by line.
func askedUnreviewed(t *testing.T) *world {
// askedReviewed is a green pull request whose Approve is asked, its review line naming the head by one reviewer.
func askedReviewed(t *testing.T) *world {
w := newWorld(t)
w.op.auto = false
p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go")
p.Title = "Fix the apostrophe (issue 948)"
p.Body = "Reviewed-by: first@shanks at " + readyHead
w.h.PullUpdated(p)
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
if open := w.op.open(); len(open) != 1 || !strings.Contains(w.op.ask(open[0]).Explanation, "not reviewed at this head") {
if open := w.op.open(); len(open) != 1 || !strings.Contains(w.op.ask(open[0]).Explanation, "reviewed by first on shanks") {
t.Fatalf("setup: the ask is %v", w.op.open())
}
return w
@@ -33,10 +34,10 @@ func edited(body string) PullEvent {
return p
}
func TestAReviewedByLineWrittenAfterTheAskWithdrawsTheUnansweredAskAndAsksAgain(t *testing.T) {
w := askedUnreviewed(t)
func TestAReviewedByLineChangedAfterTheAskWithdrawsTheUnansweredAskAndAsksAgain(t *testing.T) {
w := askedReviewed(t)
first := w.op.open()[0]
w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + readyHead))
w.h.PullEdited(edited("Reviewed-by: second@shanks at " + readyHead))
w.settleAll()
open := w.op.open()
// The heads are the same, so the key is: the ask is taken away and written again with the record in its words, which the
@@ -47,8 +48,7 @@ func TestAReviewedByLineWrittenAfterTheAskWithdrawsTheUnansweredAskAndAsksAgain(
if len(w.op.cancelled) != 0 || w.op.byKey[first].Asking != 2 {
t.Fatalf("the first ask was taken away (%v) or not written again as asking 2 (%d)", w.op.cancelled, w.op.byKey[first].Asking)
}
if e := w.op.ask(open[0]).Explanation; !strings.Contains(e, "reviewed by reviewer on shanks at "+shortOf(readyHead, 12)) ||
strings.Contains(e, "not reviewed") {
if e := w.op.ask(open[0]).Explanation; !strings.Contains(e, "reviewed by second on shanks at "+shortOf(readyHead, 12)) {
t.Fatalf("the new ask does not say the record: %s", e)
}
if len(w.forge.comments) != 0 {
@@ -63,35 +63,29 @@ func TestAReviewedByLineWrittenAfterTheAskWithdrawsTheUnansweredAskAndAsksAgain(
}
func TestAnEditThatChangesNothingTheAskSaysAsksNothingAndNoEditChangesAnAnsweredAsk(t *testing.T) {
w := askedUnreviewed(t)
// The same words: no description with a line, then a description without one.
w.h.PullEdited(edited("A new sentence in the description."))
w := askedReviewed(t)
// The same words: a description with the same line and a new sentence.
w.h.PullEdited(edited("A new sentence.\nReviewed-by: first@shanks at " + readyHead))
w.settleAll()
if w.op.count() != 1 || len(w.op.cancelled) != 0 {
t.Fatalf("an edit that changes no word of the ask asked again: %d asks, cancelled %v", w.op.count(), w.op.cancelled)
}
// An edit that brings a line for another head changes the words: the ask is made again, still saying not reviewed.
w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + strings.Repeat("a", 40)))
w.settleAll()
if w.op.count() != 2 || len(w.op.open()) != 1 {
t.Fatalf("a line for another head: %d asks, open %v", w.op.count(), w.op.open())
}
// Answered: a later edit changes nothing.
w.op.answer(w.op.open()[0], OptionApprove)
w.settleAll()
asked := w.op.count()
w.h.PullEdited(edited("Reviewed-by: reviewer@shanks at " + readyHead))
w.h.PullEdited(edited("Reviewed-by: second@shanks at " + readyHead))
w.settleAll()
if w.op.count() != asked || len(w.op.open()) != 0 {
t.Fatalf("an edit after the Approve asked again: %d asks, open %v", w.op.count(), w.op.open())
}
if d := w.delivery(IDOf("novox/app", readyHead)); d.ReviewedBy != "reviewer@shanks" {
if d := w.delivery(IDOf("novox/app", readyHead)); d.ReviewedBy != "second@shanks" {
t.Fatalf("the record is still read: %q", d.ReviewedBy)
}
}
func TestAnEditOfAnUnknownOrMergedOrEmptyPullRequestChangesNothing(t *testing.T) {
w := askedUnreviewed(t)
w := askedReviewed(t)
other := pr("novox/other", 3, readyHead, "feat/y")
other.Body = "Reviewed-by: a@b at " + readyHead
w.h.PullEdited(other) // no delivery: nothing made
@@ -173,3 +173,43 @@ func TestAGroupMergedWholeMovesSourceMainOnceAndAnnouncesWithTheTeller(t *testin
t.Fatalf("source.main moved again: %d", len(w.source.moves))
}
}
// 1106: the same refusal again and again with the trunk not moved is not asked on without end. Three refusals in a row with
// the same cause end the asking by a verdict, a status or a composed check, and say it urgent; a trunk's move, or the
// operator's Approve, starts it again.
func TestARefusalThatRepeatsWithNoTrunkMoveIsNotAskedAgainWithoutEnd(t *testing.T) {
w := answeredFirst(t)
d := w.h.deliveries[IDOf(gCat, gHk)]
refuse := func(took string) {
w.h.ForgeRefused(sdk.Forge{Kind: sdk.ForgeRefused, Repository: gCat, Number: 2, Head: gHk, Act: "merge",
Words: "novox/mesh-catalog#2 is not merged: its verdict names no trunk commit it was judged against, and main is now f70a99dbf485. " +
"The check of head d2d2d2d2d2d2 merged into trunk f70a99dbf485 was asked; the last check took " + took + ". Ask again once both statuses pass"})
}
tried := w.now
for i, took := range []string{"4m1s", "4m9s"} {
refuse(took)
d.Ready.LastTry = tried
w.h.mergeAgainNow(d)
if !d.Ready.LastTry.IsZero() {
t.Fatalf("refusal %d: the merge was not asked again at once on a verdict, as after a first refusal", i+1)
}
}
refuse("4m5s") // the third in a row, with only the check's duration changed
if d.Ready.Repeats != 3 {
t.Fatalf("%d refusals counted for the same cause, wanted 3", d.Ready.Repeats)
}
d.Ready.LastTry = tried
w.h.mergeAgainNow(d) // the composed check or a verdict answers: nothing is asked
if d.Ready.LastTry.IsZero() {
t.Fatal("the merge was asked again after the same refusal three times with the trunk not moved")
}
if lines := urgentLines(w); len(lines) != 1 || !strings.Contains(lines[0].Says, "names no trunk commit") {
t.Fatalf("the repeated refusal is not said urgent: %+v", lines)
}
// The trunk moves: the merge is asked again.
d.TrunkNow = "a1a1a1a1a1a1a1a1"
w.h.mergeAgainNow(d)
if !d.Ready.LastTry.IsZero() {
t.Fatal("a trunk's move did not let the merge be asked again")
}
}
@@ -0,0 +1,126 @@
package main
import (
"strings"
"testing"
"time"
)
// novox/hq issue 1786: two merges whose walks the controller folded, before they were cut, into a batch that was
// still queued (not yet cut into a walk) ended superseded: the batch is no walk that answers a merge, so the follow
// loop took the walk's taken_over_by off ("the walk that took its own over does not answer it") and the table's
// supersession then held ("names no walk that took it over"). Delivered by the batch's walk, they showed
// superseded for good. A batch that took a walk over is waited for; and one already ended so follows its taker.
const (
head1786 = "1786178617861786178617861786178617861786"
merge1786 = "a1786a1786a1786a1786a1786a1786a1786a17860"
)
// folded1786 is a delivery whose walk was folded into a batch that is still queued.
func folded1786(t *testing.T) (*world, string) {
t.Helper()
w := newWorld(t)
id := IDOf("novox/app", head1786)
w.h.PullUpdated(pr("novox/app", 7, head1786, "feat/x", "src/a.go"))
w.h.Checked(verdict("novox/app", 7, head1786, "pass"))
w.h.PullMerged(merged("novox/app", 7, head1786, merge1786))
w.h.WalkMoved(w.ctl.put(aWalk("plan-own", "novox/app", merge1786, true)))
w.settleAll()
if s := w.state(id); s != Delivering {
t.Fatalf("setup: %s, want delivering", s)
}
closes := w.now.Add(time.Minute)
batch := Walk{ID: "plan-batch", Repository: "novox/other", Branch: "main", Commit: "ffff", Created: w.now, State: walkQueued,
Delivery: &WalkDelivery{Awaits: "mesh-delivery", Batch: &WalkBatch{ClosesAt: &closes},
Merges: []WalkMerge{{Repository: "novox/app", Commit: merge1786}}}}
w.h.WalkMoved(w.ctl.put(batch))
folded := w.ctl.walk("plan-own")
folded.State = walkSuperseded
folded.Note = "folded into plan-batch before it was cut"
folded.Delivery.TakenOverBy = "plan-batch"
w.h.WalkMoved(w.ctl.put(folded))
w.settleAll()
return w, id
}
func cut1786(w *world, state string) {
b := w.ctl.walk("plan-batch")
b.State = state
b.Delivery.Batch = nil
b.Tiers, b.Modules = [][]string{{"app"}}, map[string]*WalkModule{"app": {}}
w.h.WalkMoved(w.ctl.put(b))
w.settleAll()
}
func TestAWalkFoldedIntoAQueuedBatchIsNotSupersededAndFollowsTheBatchWhenItIsCut(t *testing.T) {
w, id := folded1786(t)
if d := w.delivery(id); d.State.Final() || d.Walk.TakenOverBy != "plan-batch" {
t.Fatalf("folded into a queued batch it is %s, taken over by %q; wanted it still following plan-batch", d.State, d.Walk.TakenOverBy)
}
cut1786(w, walkBuilding)
cut1786(w, walkDone)
if s := w.state(id); s != Delivered {
t.Fatalf("the batch's walk is done and it is %s; wanted delivered", s)
}
}
func TestADeliverySupersededWhileItsTakerWasABatchIsHealedWhenTheTakerEndsDone(t *testing.T) {
w, id := folded1786(t)
d := w.h.deliveries[id]
// The state kept by the fault: superseded, its walk record naming the walk that took it over.
d.State = Superseded
d.Transitions = append(d.Transitions, Transition{At: w.now, From: Delivering, To: Superseded, Event: EvSuperseded})
if err := w.h.Store.PutDelivery(d); err != nil {
t.Fatal(err)
}
cut1786(w, walkBuilding)
cut1786(w, walkDone)
w.later(time30s)
w.settleAll()
d = w.delivery(id)
// A final state stays final (ADR 0239 §5): the taker's end is kept as DeliveredBy, as issue 442 and 747 do.
if d.State != Superseded || d.DeliveredBy == "" || !memberReached(d) || !d.coveredStop() {
t.Fatalf("superseded by the fault and its taker done, it is %s, delivered by %q, reached %v; wanted superseded, delivered by the taker, reached",
d.State, d.DeliveredBy, memberReached(d))
}
if st, _ := statusOf(d); st != "success" {
t.Fatalf("its status is %s; wanted success", st)
}
}
func TestAFinalDeliveryWhoseTakerDoesNotAnswerItIsNeitherHealedNorLoggedEveryPass(t *testing.T) {
w, id := folded1786(t)
d := w.h.deliveries[id]
d.State = Superseded
d.Transitions = append(d.Transitions, Transition{At: w.now, From: Delivering, To: Superseded, Event: EvSuperseded})
other := aWalk("plan-batch", "novox/other", "ffff", false)
other.State = walkDone
w.h.WalkMoved(w.ctl.put(other))
w.settleAll()
before := w.logs()
w.later(time30s)
w.settleAll()
if d := w.delivery(id); d.DeliveredBy != "" || d.State != Superseded {
t.Fatalf("healed by a walk that does not answer it: %s %q", d.State, d.DeliveredBy)
}
if n := strings.Count(w.logs(), "does not answer its merge") - strings.Count(before, "does not answer its merge"); n != 0 {
t.Fatalf("logged %d times on a pass over a final record", n)
}
}
func TestASupersededDeliveryWithNoWalkThatTookItOverStaysSuperseded(t *testing.T) {
w := newWorld(t)
id := IDOf("novox/app", head1786)
w.h.PullUpdated(pr("novox/app", 7, head1786, "feat/x", "src/a.go"))
w.h.PullUpdated(pr("novox/app", 7, "b"+head1786[1:], "feat/x", "src/a.go"))
w.settleAll()
if s := w.state(id); s != Superseded {
t.Fatalf("a head superseded by a newer one is %s", s)
}
w.later(time30s)
w.settleAll()
if s := w.state(id); s != Superseded {
t.Fatalf("it moved: %s", s)
}
}
@@ -0,0 +1,120 @@
package main
import "testing"
const gHk2 = "f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4f4"
// Issue 1792: a sibling's new head put a member's own mesh/repo-check back to pending ("out of date: judged beside ..."),
// and nothing asked it again. The composed check passed, the group stayed held ("unasked") on the member's pending
// status, and the operator's Approve was never asked. The group's hold on a member's own pending repository check asks
// that check, once for the head, with no composed check (ADR 0382 §2: each member's own check stays required).
func TestAGroupHeldBeforeItsAskOnAMembersOwnPendingRepoCheckAsksThatCheckOnce(t *testing.T) {
w := newWorld(t)
w.op.auto = false
w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go"))
w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json"))
w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0))
w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0))
w.settleAll()
g := w.h.groups["feat/g"]
setChecks(w, gCat, gHk, "success", "success")
setChecks(w, gCtl, gHc, "success", "pending")
asked := len(w.ctl.checks)
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed",
Members: []struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Commit string `json:"commit"`
Trunk string `json:"trunk,omitempty"`
}{{Owner: "novox", Repo: "mesh-controller", Commit: gHc, Trunk: gTrunk0[:12]},
{Owner: "novox", Repo: "mesh-catalog", Commit: gHk, Trunk: gTrunk0[:12]}}})
w.settleAll()
w.later(time30s)
w.settleAll()
own, composed := ownChecks(w, asked)
if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) {
t.Fatalf("own checks %v, composed %v; wanted the member's own check asked once", own, composed)
}
if w.op.count() != 0 {
t.Fatalf("the operator was asked with a required status pending: %d", w.op.count())
}
}
func composedPassAt(w *world, hc, hk string) {
g := w.h.groups["feat/g"]
w.h.Checked(CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed",
Members: []struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Commit string `json:"commit"`
Trunk string `json:"trunk,omitempty"`
}{{Owner: "novox", Repo: "mesh-controller", Commit: hc, Trunk: gTrunk0[:12]},
{Owner: "novox", Repo: "mesh-catalog", Commit: hk, Trunk: gTrunk0[:12]}}})
}
// heldOnOwnPending is the group of issue 1792 after its first own ask: the member's own check asked, once.
func heldOnOwnPending(t *testing.T) *world {
w := newWorld(t)
w.op.auto = false
w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go"))
w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json"))
w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0))
w.h.Checked(judged(gCat, 2, gHk, "pass", gTrunk0))
w.settleAll()
setChecks(w, gCat, gHk, "success", "success")
setChecks(w, gCtl, gHc, "success", "pending")
asked := len(w.ctl.checks)
composedPassAt(w, gHc, gHk)
w.settleAll()
w.later(time30s)
w.settleAll()
if own, _ := ownChecks(w, asked); len(own) != 1 {
t.Fatalf("the first own ask: %v", own)
}
return w
}
// A second sibling push puts the member's repo-check out of date again after its own check answered: asked again.
func TestAMembersOwnCheckIsAskedAgainWhenASecondSiblingHeadPutsItOutOfDate(t *testing.T) {
w := heldOnOwnPending(t)
w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0)) // its own check answered
setChecks(w, gCtl, gHc, "success", "success")
w.settleAll()
w.h.PullUpdated(pr(gCat, 2, gHk2, "feat/g", "modules/app/module.json"))
w.h.Checked(judged(gCat, 2, gHk2, "pass", gTrunk0))
w.settleAll()
setChecks(w, gCat, gHk2, "success", "success")
setChecks(w, gCtl, gHc, "success", "pending")
asked := len(w.ctl.checks)
composedPassAt(w, gHc, gHk2)
w.settleAll()
w.later(time30s)
w.settleAll()
own, composed := ownChecks(w, asked)
if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) {
t.Fatalf("own %v composed %v; wanted the member's own check asked again, once", own, composed)
}
}
// An own ask before the Approve, answered, and then a merge refused for the member's own pending status at the same head:
// asked again (the one ask is per answer, not per head's life).
func TestAMergeRefusedForOwnPendingAfterAnAnsweredOwnAskAsksItAgain(t *testing.T) {
w := heldOnOwnPending(t)
w.h.Checked(judged(gCtl, 1, gHc, "pass", gTrunk0))
setChecks(w, gCtl, gHc, "success", "success")
w.later(time30s)
w.settleAll()
if len(w.op.open()) != 1 {
t.Fatalf("the group is not asked once its statuses are green: %d", w.op.count())
}
w.forge.refuseRepo = map[string]error{gCtl: ownPending}
w.op.answer(w.op.open()[0], OptionApprove)
w.settleAll()
w.h.PullMerged(merged(gCat, 2, gHk, gT1))
asked := len(w.ctl.checks)
w.settleAll()
own, composed := ownChecks(w, asked)
if len(composed) != 0 || len(own) != 1 || own[0] != ":"+IDOf(gCtl, gHc) {
t.Fatalf("own %v composed %v; wanted the member's own check asked again after the refusal", own, composed)
}
}
@@ -0,0 +1,223 @@
package main
import (
"strings"
"testing"
)
// novox/hq issue 1803, ADR 0496: the Approve is asked only for a head with a review recorded at that head. A missing
// line, `Reviewed-by: pending` or a line naming another commit holds the ask; the delivery stays ready and says it waits
// for a review at the head. An edit naming the head asks at once. A group asks when every member is reviewed. An ask
// already raised is not touched.
const stale1803 = "9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f9f"
// greenWithBody is a green pull request whose description is body, the operator not answering by himself.
func greenWithBody(t *testing.T, body string) *world {
w := newWorld(t)
w.op.auto = false
p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go")
p.Title = "Fix the apostrophe (issue 948)"
p.Body = body
w.h.PullUpdated(p)
w.h.Checked(judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1"))
w.forge.open = []OpenPull{openPull("novox/app", 7, readyHead)}
w.h.SeeOpenPulls()
w.settleAll()
return w
}
func (w *world) wantHeld(t *testing.T, label string) {
t.Helper()
want := "waits for a review at " + shortOf(readyHead, 12)
d := w.delivery(IDOf("novox/app", readyHead))
if w.op.count() != 0 || len(w.op.open()) != 0 {
t.Fatalf("%s: the operator was asked (%d asks)", label, w.op.count())
}
if d.State != Ready || d.Ready == nil || !strings.Contains(d.Ready.Waits, want) {
t.Fatalf("%s: state %s, waits %+v; wanted ready and %q", label, d.State, d.Ready, want)
}
found := false
for k, v := range w.forge.statuses {
if strings.HasSuffix(strings.SplitN(k, " ", 2)[1], "mesh/delivery") && strings.Contains(v, want) {
found = true
}
}
if !found {
t.Fatalf("%s: no mesh/delivery status says %q: %v", label, want, w.forge.statuses)
}
}
func TestA1803PendingReviewLineHoldsTheAsk(t *testing.T) {
w := greenWithBody(t, "Reviewed-by: pending")
w.wantHeld(t, "1803/AC1 pending")
for range 3 {
w.later(2 * 60 * 1e9)
w.settleAll()
}
w.wantHeld(t, "1803/AC1 pending after ticks")
}
func TestA1803MissingReviewLineHoldsTheAsk(t *testing.T) {
w := greenWithBody(t, "No review line at all.")
w.wantHeld(t, "1803/AC1 missing")
}
func TestA1803ReviewLineOfAnotherCommitHoldsTheAsk(t *testing.T) {
w := greenWithBody(t, "Reviewed-by: reviewer@shanks at "+stale1803)
w.wantHeld(t, "1803/AC2 stale")
}
func TestA1803ReviewLineNamingTheHeadIsAsked(t *testing.T) {
w := greenWithBody(t, "Reviewed-by: reviewer@shanks at "+readyHead)
if len(w.op.open()) != 1 {
t.Fatalf("1803/AC3: a review at the head is not asked: %v", w.op.open())
}
if e := w.op.ask(w.op.open()[0]).Explanation; !strings.Contains(e, "reviewed by reviewer on shanks") {
t.Fatalf("the ask does not name the review: %s", e)
}
}
func TestA1803AnEditNamingTheHeadRaisesTheAskAtOnce(t *testing.T) {
w := greenWithBody(t, "Reviewed-by: pending")
w.wantHeld(t, "1803/AC4 before the edit")
e := pr("novox/app", 7, readyHead, "feat/x")
e.Body = "Reviewed-by: reviewer@shanks at " + readyHead
w.h.PullEdited(e)
w.settleAll()
if len(w.op.open()) != 1 {
t.Fatalf("1803/AC4: the edit naming the head raised no ask: %d asks", w.op.count())
}
// An unanswered ask whose review is taken back is withdrawn: the hold applies again (ADR 0496 §1).
e.Body = "Reviewed-by: pending"
w.h.PullEdited(e)
w.settleAll()
if len(w.op.open()) != 0 {
t.Fatalf("1803/AC4: an unanswered ask stayed after its review was taken back: open %v", w.op.open())
}
}
func TestA1803AnApproveAlreadyGivenIsNotAffected(t *testing.T) {
w := greenWithBody(t, "Reviewed-by: reviewer@shanks at "+readyHead)
w.op.answer(w.op.open()[0], OptionApprove)
w.settleAll()
if len(w.forge.merges) != 1 {
t.Fatalf("1803/AC6: the approved head was not merged: %v", w.forge.merges)
}
// A later edit to pending changes nothing of the Approve given.
e := pr("novox/app", 7, readyHead, "feat/x")
e.Body = "Reviewed-by: pending"
w.h.PullEdited(e)
w.settleAll()
if d := w.delivery(IDOf("novox/app", readyHead)); d.Ready == nil || d.Ready.Approved == nil {
t.Fatalf("1803/AC6: the Approve given was lost: %+v", d.Ready)
}
}
const gTools, gHt = "novox/mesh-tools", "e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3e3"
// groupOf is a green group of the given members (repository, number, head, review body), composed green, the operator not
// answering by himself.
func groupOf(t *testing.T, members [][4]string) *world {
w := newWorld(t)
w.op.auto = false
for i, m := range members {
p := pr(m[0], i+1, m[2], "feat/g", "src/a.go")
p.Body = m[3]
w.h.PullUpdated(p)
}
for i, m := range members {
w.h.Checked(judged(m[0], i+1, m[2], "pass", gTrunk0))
}
w.settleAll()
g := w.h.groups["feat/g"]
if g == nil || g.Check == nil {
t.Fatalf("no composed check was asked: %+v", g)
}
c := CheckedEvent{Group: "feat/g", ID: g.Check.Asked, Verdict: "pass", Summary: "composed"}
for _, m := range members {
owner, repo, _ := strings.Cut(m[0], "/")
c.Members = append(c.Members, struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Commit string `json:"commit"`
Trunk string `json:"trunk,omitempty"`
}{Owner: owner, Repo: repo, Commit: m[2], Trunk: gTrunk0[:12]})
}
w.h.Checked(c)
w.settleAll()
return w
}
func reviewedAt(head string) string { return "Reviewed-by: reviewer@shanks at " + head }
// wantGroupSays checks the group's state, its mesh/delivery-group status and the member's mesh/delivery status.
func (w *world) wantGroupSays(t *testing.T, label, member, head string, want, notWant string) {
t.Helper()
g := w.h.groups["feat/g"]
state, why, held := w.h.groupState(g, w.h.membersOf(g))
if state != "ready" || !held || !strings.Contains(why, want) || (notWant != "" && strings.Contains(why, notWant)) {
t.Fatalf("%s: group state %s, why %q, held %v; wanted ready waiting with %q and without %q", label, state, why, held, want, notWant)
}
for _, c := range []string{"mesh/delivery-group", "mesh/delivery"} {
said := w.forge.statuses[member+"@"+head+" "+c]
if !strings.HasPrefix(said, "pending") || !strings.Contains(said, want) || (notWant != "" && strings.Contains(said, notWant)) {
t.Fatalf("%s: %s on %s says %q; wanted pending with %q and without %q", label, c, member, said, want, notWant)
}
}
}
func TestA1803AGroupIsAskedOnlyWhenEveryMemberIsReviewedAtItsHead(t *testing.T) {
w := groupOf(t, [][4]string{{gCtl, "", gHc, reviewedAt(gHc)}, {gCat, "", gHk, "Reviewed-by: pending"}})
if w.op.count() != 0 {
t.Fatalf("1803/AC5: a group with an unreviewed member was asked: %d asks", w.op.count())
}
w.wantGroupSays(t, "1803/AC5", gCtl, gHc, "mesh-catalog#2", "mesh-controller#1")
e := pr(gCat, 2, gHk, "feat/g")
e.Body = reviewedAt(gHk)
w.h.PullEdited(e)
w.settleAll()
if len(w.op.open()) != 1 {
t.Fatalf("1803/AC5: every member reviewed, yet %d asks", w.op.count())
}
if said := w.forge.statuses[gCtl+"@"+gHc+" mesh/delivery-group"]; !strings.HasPrefix(said, "success") {
t.Fatalf("1803/AC5: the group's status once asked is %q", said)
}
}
func TestA1803TheWaitingWordsFollowTheMembersStillWaiting(t *testing.T) {
w := groupOf(t, [][4]string{{gCtl, "", gHc, reviewedAt(gHc)}, {gCat, "", gHk, "Reviewed-by: pending"}, {gTools, "", gHt, "Reviewed-by: pending"}})
w.wantGroupSays(t, "1803/AC7 two waiting", gCtl, gHc, "mesh-catalog#2", "")
w.wantGroupSays(t, "1803/AC7 two waiting", gCtl, gHc, "mesh-tools#3", "")
e := pr(gCat, 2, gHk, "feat/g")
e.Body = reviewedAt(gHk)
w.h.PullEdited(e)
w.settleAll()
if w.op.count() != 0 {
t.Fatalf("1803/AC7: a member still waits, yet %d asks", w.op.count())
}
// Only the other member is named now, on the group and on every member's own status.
w.wantGroupSays(t, "1803/AC7 one waiting", gCtl, gHc, "mesh-tools#3", "mesh-catalog#2")
w.wantGroupSays(t, "1803/AC7 one waiting", gCat, gHk, "mesh-tools#3", "mesh-catalog#2")
}
func TestA1803APushAfterAnApprovingReviewHoldsTheNewHead(t *testing.T) {
w := greenWithBody(t, reviewedAt(readyHead))
if len(w.op.open()) != 1 {
t.Fatalf("setup: the reviewed head is not asked")
}
// A new head on the same pull request whose description still names the old commit.
const pushed2 = "7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a"
p := pr("novox/app", 7, pushed2, "feat/x", "src/a.go")
p.Body = reviewedAt(readyHead)
w.h.PullUpdated(p)
w.h.Checked(judgedAt("novox/app", 7, pushed2, "pass", "t1t1t1t1t1t1"))
w.settleAll()
d := w.delivery(IDOf("novox/app", pushed2))
if d.Ready == nil || !strings.Contains(d.Ready.Waits, waitsForReview+shortOf(pushed2, 12)) {
t.Fatalf("1803/AC8: the new head waits %+v; wanted it to wait for a review at %s", d.Ready, shortOf(pushed2, 12))
}
if len(w.op.open()) != 0 {
t.Fatalf("1803/AC8: the new head was asked on the old review: %v", w.op.open())
}
}
@@ -287,9 +287,20 @@ func (h *Holder) ForgeRefused(f sdk.Forge) {
if r.RefusedSince == nil {
r.RefusedSince = &now
}
r.noteRefusal(d, answer)
if !outOfDateRefusal(answer) {
r.Urgent = true // never a warning again for this head
}
if r.Repeats >= maxRepeatedRefusals {
// Loud, not endless (novox/hq issue 1106): the same refusal again with the trunk not moved is no trunk's move, so
// the merge is not asked again by a verdict or a status; the operator is told, urgent, and decides.
r.Urgent = true
if r.Repeats == maxRepeatedRefusals {
h.Logf("[mesh-delivery] %s: the merge tool refused it %d times in a row with the same words and the trunk not moved, so "+
"the merge is asked no more until the trunk moves, a new head is announced or the operator approves again: %s",
d.ID, r.Repeats, answer)
}
}
if answer != r.Answer {
h.Logf("[mesh-delivery] %s is approved, and the merge tool did not merge it (asked again at the next verdict, Approve or composed check): %s", d.ID,
answer)
@@ -132,7 +132,7 @@ type Delivery struct {
Ready *ReadyRun `json:"ready,omitempty"`
// ReviewedBy and ReviewedAt are the review record its pull request's description holds: the line
// `Reviewed-by: <who> at <full sha>` (novox/hq ADR 0421 §5, issue 1022), as written. Read at every head; never a
// gate: the ask says first when the line is missing or names another head.
// hold: no Approve is asked until it names the head (novox/hq ADR 0496, issue 1803).
ReviewedBy string `json:"reviewed_by,omitempty"`
ReviewedAt string `json:"reviewed_at,omitempty"`
// FreshWanted is whether a passing verdict that arrived out of date is to have its check asked against the
@@ -351,7 +351,7 @@ func (d *Delivery) coveredStop() bool {
switch d.State {
case Stopped:
return d.Stranded != "" && d.CoveredBy != "" || d.DeliveredBy != ""
case Failed:
case Failed, Superseded:
return d.DeliveredBy != ""
}
return false
@@ -397,6 +397,9 @@ func (d *Delivery) strandedEndedSaid() string {
// coveredStopSaid is how a covered stop or failure reads: on the forge's status, in its view and note, and in `show`.
func (d *Delivery) coveredStopSaid() string {
if d.State == Superseded {
return "delivered after its walk was superseded: " + d.DeliveredBy
}
ended := "its stop"
if d.State == Failed {
ended = "its failure"
@@ -496,7 +499,7 @@ func ReviewIn(body string) (who, sha string) {
}
// reviewWords is what the delivery's review record says of its head, for the ask: reviewed by whom, or, first, that it
// is not reviewed at this head (never a gate, ADR 0421 §5). ok is whether it is reviewed at this head.
// is not reviewed at this head (the ask is held then, ADR 0496). ok is whether it is reviewed at this head.
func reviewWords(d *Delivery) (words string, ok bool) {
switch {
case d.ReviewedBy == "":
@@ -1016,7 +1019,10 @@ type Group struct {
Hold *GroupHold `json:"hold,omitempty"`
// SaidStatus is the mesh/delivery-group status last written for it: a change of the status a state takes is
// written too, though the state did not change (novox/hq issue 902).
SaidStatus string `json:"said_status,omitempty"`
SaidStatus string `json:"said_status,omitempty"`
// SaidReview is the wait for a review last said of the group while it was held for one (ADR 0496 §2): a change of
// the members still waiting is said too, though the state did not change.
SaidReview string `json:"said_review,omitempty"`
Owed []Effect `json:"owed,omitempty"`
}
@@ -1071,3 +1077,13 @@ func mustJSON(v any) json.RawMessage {
raw, _ := json.Marshal(v)
return raw
}
// supersededByItsWalk is whether a delivery ended superseded because its walk was: its last move was a supersession
// from published or delivering, not a newer head of its pull request's (issue 1786).
func (d *Delivery) supersededByItsWalk() bool {
if d.State != Superseded || len(d.Transitions) == 0 {
return false
}
t := d.Transitions[len(d.Transitions)-1]
return t.Event == EvSuperseded && (t.From == Published || t.From == Delivering)
}
@@ -85,6 +85,11 @@ type ReadyRun struct {
// Urgent is set once a refusal of this head was one that is urgent at once: its severity never steps down to a
// warning after (issue 1275, AC6).
Urgent bool `json:"urgent,omitempty"`
// Repeats is how many times in a row the merge tool refused with the same words while the trunk stayed where RepeatTrunk
// says: past maxRepeatedRefusals the merge is asked no more by a verdict, a status or a composed check, and the refusal
// is urgent (novox/hq issue 1106). A trunk's move, a new head or the operator's Approve starts the count again.
Repeats int `json:"repeats,omitempty"`
RepeatTrunk string `json:"repeat_trunk,omitempty"`
// RefusedSince is when the merge tool began refusing it without a break.
RefusedSince *time.Time `json:"refused_since,omitempty"`
// Merged is when the merge tool merged it.
@@ -214,7 +219,14 @@ func (h *Holder) readyAsks(now time.Time) []func() {
r.Failed = ""
waits := h.readyWaits(d)
if waits != r.Waits {
wasHeld := d.holdsForReview()
r.Waits = waits
if (wasHeld || d.holdsForReview()) && d.shownOnItsPullRequest() {
// The status and the view say what it waits for, again whenever the words change while held, and no more once asked.
state, desc := statusOf(d)
d.owe(Effect{Kind: EffectView, Since: now})
d.owe(Effect{Kind: EffectStatus, Commit: d.Commit, Context: "mesh/delivery", State: state, Line: desc, Since: now})
}
if waits != "" {
h.Logf("[mesh-delivery] %s waits: %s", d.ID, waits)
}
@@ -322,10 +334,43 @@ func (h *Holder) askFresh(d *Delivery, now time.Time) func() {
// verdict, an Approve, a group's composed check), so the wait after a refusal does not hold it. Under the lock.
func (h *Holder) mergeAgainNow(d *Delivery) {
if d != nil && d.Ready != nil && d.Ready.Merged == nil {
if d.Ready.repeatedRefusal(d) {
return // the same refusal again and again with the trunk where it was: asking more changes nothing (issue 1106)
}
d.Ready.LastTry = time.Time{}
}
}
// maxRepeatedRefusals is how many refusals in a row, with the same words and the trunk not moved, end the asking again: a
// refusal that repeats with no trunk move is not a trunk's move, and the check asked after each only brings the same words.
const maxRepeatedRefusals = 3
// repeatedRefusal is whether the merge was refused maxRepeatedRefusals times with these words and the trunk where it is
// now: it is then asked no more until something changes (a trunk's move, a new head, the operator's Approve).
func (r *ReadyRun) repeatedRefusal(d *Delivery) bool {
return r.Repeats >= maxRepeatedRefusals && r.RepeatTrunk == d.TrunkNow
}
// refusalKey is a refusal's first sentence, up to the trunk it names ("... and main is now <t>"): what follows says how long the
// last check took and the like, which differs each time and is no change of cause.
func refusalKey(answer string) string {
if i := strings.Index(answer, " is now "); i >= 0 {
if j := strings.Index(answer[i:], ". "); j >= 0 {
return answer[:i+j]
}
}
return answer
}
// noteRefusal counts a refusal: the same words with the trunk where it was count on, anything else starts the count at one.
func (r *ReadyRun) noteRefusal(d *Delivery, answer string) {
if refusalKey(answer) == refusalKey(r.Answer) && r.RepeatTrunk == d.TrunkNow && r.Repeats > 0 {
r.Repeats++
return
}
r.Repeats, r.RepeatTrunk = 1, d.TrunkNow
}
// readyWaits is what a delivery's merge waits on; empty when the Approve may be asked and the merge asked. A ready
// delivery is asked. So is a proposed one whose verdict went out of date, when an Approve was asked for it already (it
// stays through a trunk move, ADR 0421 §3) and its check against the trunk's head is not on its way; one without an
@@ -343,12 +388,48 @@ func (h *Holder) readyWaits(d *Delivery) string {
default:
return "its check has not passed yet (" + string(d.State) + ")"
}
// ADR 0496 §1, §2: the Approve is asked only for heads with a review recorded at them. An ask already raised or
// answered is not touched (covering).
if h.covering(d) == nil {
if w := h.reviewWaits(d); w != "" {
return w
}
}
if d.Group == "" {
return ""
}
return h.groupTurnToMerge(d)
}
// waitsForReview is how a hold for a review begins, as the delivery's state and its mesh/delivery status say it.
const waitsForReview = "waits for a review at "
// reviewWaits is why the Approve is not asked yet for want of a review (novox/hq ADR 0496, issue 1803): a delivery
// whose description holds no review line, `pending` or a line naming another commit, and for a group any member so.
// Empty when every delivery the ask covers is reviewed at its head. Under the lock.
func (h *Holder) reviewWaits(d *Delivery) string {
members := h.askMembers(d)
var waiting []string
for _, m := range members {
if _, ok := reviewWords(m); ok {
continue
}
if len(members) == 1 {
return waitsForReview + shortOf(m.Commit, 12)
}
waiting = append(waiting, "at "+shortOf(m.Commit, 12)+" ("+pullSaid(m)+")")
}
if len(waiting) == 0 {
return ""
}
return waitsForReview + strings.Join(waiting, " and ")[len("at "):]
}
// holdsForReview is whether a delivery's wait is the hold for a review.
func (d *Delivery) holdsForReview() bool {
return d.Ready != nil && strings.HasPrefix(d.Ready.Waits, waitsForReview)
}
// groupTurnToMerge is why a group member may not merge yet; empty when it may: the group's composed check passed for
// its members' heads as they are and is not out of date. No member waits for another's merge or delivery (novox/hq
// ADR 0418 §4): once the composed check passed and the Approve covers the heads, every member is merged in one pass.
@@ -134,14 +134,14 @@ func (h *Holder) Show(id string) (any, error) {
out["reads"] = d.strandedEndedSaid() // stopped, and nothing can carry what it waited for (issue 1581)
}
if g := h.groups[d.Group]; g != nil {
state, why := GroupState(g, h.membersOf(g))
state, why, _ := h.groupState(g, h.membersOf(g))
out["group"] = map[string]any{"group": g, "state": state, "why": why}
}
return out, nil
}
if g := h.groups[id]; g != nil {
members := h.membersOf(g)
state, why := GroupState(g, members)
state, why, _ := h.groupState(g, members)
return map[string]any{"group": g, "state": state, "why": why, "members": members}, nil
}
return nil, fmt.Errorf("no delivery or group %q: `deliveries` lists them", id)
@@ -184,7 +184,7 @@ func (h *Holder) Groups(all bool) []GroupLine {
if len(members) == 0 {
continue
}
state, why := GroupState(g, members)
state, why, _ := h.groupState(g, members)
if !all && (state == "delivered" || state == "failed" || state == "stopped") {
continue
}
@@ -119,6 +119,9 @@ type Client struct {
Templates *templateStore
// Now dates the reports; nil is time.Now.
Now func() time.Time
// SocketUp wakes the client when the node-engine's write socket is made again (watchSocket); nil never does.
// A module whose write failed for want of the socket is filled again then (issue 1165), and nothing else is.
SocketUp <-chan struct{}
// watchBound watches a provider's bucket of entries for this machine's keys, once per run (consumers.go); set by
// Run.
@@ -147,6 +150,8 @@ type state struct {
// of the consumers left out of one of its files.
providing map[string]bool
leftOut map[string]string
// unplaced is a module whose last fill failed for want of the node-engine's write socket (issue 1165).
unplaced map[string]bool
// bound is each provider's bucket of entries this client watches, by key: this machine's consumers' entries alone;
// boundReady is a bucket whose current entries are all read, and boundUsers the modules filled from each.
bound map[string]map[string][]byte
@@ -161,6 +166,8 @@ type update struct {
changed []byte
// asked is the tool runner asking for a module's value (asks.go). Set, entry is nil.
asked *nats.Msg
// socketUp is the node-engine's write socket made again (issue 1165). Set, entry is nil.
socketUp bool
}
// Run serves until ctx ends or a watch stops, which it returns: run again, it starts from the current values.
@@ -248,6 +255,23 @@ func (c *Client) Run(ctx context.Context) error {
return fmt.Errorf("listening for %s: %w", SecretChangedSubject, err)
}
defer func() { _ = said.Unsubscribe() }()
// The node-engine's write socket made again: what failed for want of it is filled again, in the loop below.
if c.SocketUp != nil {
go func() {
for {
select {
case <-ctx.Done():
return
case <-c.SocketUp:
select {
case updates <- update{socketUp: true}:
case <-ctx.Done():
return
}
}
}
}()
}
// This machine's tool runner asking for a module's config value (asks.go): answered from this run's state, in
// its loop, so an answer is never read from half an update.
asks, err := c.Conn.Subscribe(AskSubject(c.Node), func(m *nats.Msg) {
@@ -266,7 +290,7 @@ func (c *Client) Run(ctx context.Context) error {
reported: map[string][]byte{}, secrets: map[string]heldSecret{}, items: map[string]map[string]itemValue{},
recorded: map[string][]byte{}, assigned: map[string]bool{}, binds: map[string]binding{},
providing: map[string]bool{}, leftOut: map[string]string{}, bound: map[string]map[string][]byte{},
boundReady: map[string]bool{}, boundUsers: map[string]map[string]bool{}}
boundReady: map[string]bool{}, unplaced: map[string]bool{}, boundUsers: map[string]map[string]bool{}}
// Every current value first: nothing is filled from half of them.
current := map[string]bool{}
var early []*nats.Msg
@@ -282,6 +306,9 @@ func (c *Client) Run(ctx context.Context) error {
early = append(early, u.asked)
continue
}
if u.socketUp {
continue // the first fill below tries every file
}
if u.changed != nil {
// Nothing is held yet: every secret is asked for when it is first filled.
continue
@@ -326,6 +353,10 @@ func (c *Client) Run(ctx context.Context) error {
switch {
case u.asked != nil:
c.respond(s, u.asked)
case u.socketUp:
for m := range s.unplaced {
dirty[m] = true
}
case u.changed != nil:
for _, m := range c.secretChanged(s, u.changed) {
dirty[m] = true
@@ -550,6 +581,7 @@ func (c *Client) leftModules(s *state) []string {
// leave removes a module's needs and its report: it is no longer the provisioner's on this node.
func (c *Client) leave(ctx context.Context, kvs map[string]jetstream.KeyValue, s *state, module string) {
prefix := c.Node + "." + module + "."
delete(s.unplaced, module)
removed := 0
for _, key := range sortedKeys(s.needs) {
if strings.HasPrefix(key, prefix) {
@@ -660,6 +692,7 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu
c.forgetItems(ctx, kvs, s, module, keep)
}
moved := false
delete(s.unplaced, module)
for _, f := range targets {
// File by file (issue 1553, ADR 0349 clause 4): a target with a value that waits keeps the file there is.
if !allKnown(f.text, values) {
@@ -672,6 +705,9 @@ func (c *Client) reconcile(ctx context.Context, kvs map[string]jetstream.KeyValu
err = errFill(a)
}
if err != nil {
if socketAbsent(err) {
s.unplaced[module] = true
}
failed = append(failed, fmt.Sprintf("%s: %v", f.Path, err))
continue
}
@@ -200,6 +200,14 @@ type engine struct {
}
func startEngine(t *testing.T, owners map[string]string) *engine {
t.Helper()
e := newEngine(t, owners)
e.listen(t)
return e
}
// newEngine is an engine whose socket is not made yet: a node-engine not started, or replacing itself.
func newEngine(t *testing.T, owners map[string]string) *engine {
t.Helper()
// A unix socket's path is short: under /tmp, not the test's own directory.
dir, err := os.MkdirTemp("/tmp", "p5-")
@@ -209,6 +217,12 @@ func startEngine(t *testing.T, owners map[string]string) *engine {
t.Cleanup(func() { os.RemoveAll(dir) })
e := &engine{root: t.TempDir(), socket: filepath.Join(dir, "fill.sock"), owner: owners,
moved: map[string]int{}, asked: map[string]int{}, wrote: map[string]string{}}
return e
}
// listen makes the socket and answers on it until stop, which removes it as a node-engine stopping does.
func (e *engine) listen(t *testing.T) (stop func()) {
t.Helper()
l, err := net.Listen("unix", e.socket)
if err != nil {
t.Fatal(err)
@@ -233,7 +247,7 @@ func startEngine(t *testing.T, owners map[string]string) *engine {
}()
}
}()
return e
return func() { l.Close(); os.Remove(e.socket) }
}
func (e *engine) at(path string) string { return filepath.Join(e.root, path) }
@@ -349,10 +363,14 @@ func aClient(t *testing.T, b *aBus, e *engine, state string) (*Client, func()) {
t.Fatal(err)
}
secrets.Within = 2 * time.Second
c := &Client{Node: node, JS: js, Conn: nc, Secrets: secrets,
ctx, cancel := context.WithCancel(context.Background())
up, err := watchSocket(ctx, e.socket)
if err != nil {
t.Fatal(err)
}
c := &Client{Node: node, JS: js, Conn: nc, Secrets: secrets, SocketUp: up,
Fill: SocketFiller{Socket: e.socket, Busy: 10 * time.Millisecond, Tries: 3},
Templates: &templateStore{dir: state, fetch: HTTPFetch}}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- c.Run(ctx) }()
stopped := false
@@ -703,3 +721,23 @@ func TestNatsResticsFilesFillFromTheBackupPlacement(t *testing.T) {
b.assign(t, "restic", plan("# a\n/srv/a\n", "# a\nstate\n"))
eventually(t, "a module taken away refills backups.conf", func() bool { return e.read(t, backups) == head+"# a\n/srv/a\n" })
}
// defaults writes the defaults a module's manifest declares as mesh-config resolves them (source default), for a
// moved module whose templates read config values (issue 1780).
func (b *aBus) defaults(t *testing.T, module string, manifest []byte) {
t.Helper()
var m struct {
Config map[string]struct {
Default json.RawMessage `json:"default"`
} `json:"config"`
}
if err := json.Unmarshal(manifest, &m); err != nil {
t.Fatal(err)
}
for key, d := range m.Config {
if len(d.Default) > 0 && string(d.Default) != "null" {
b.put(t, ConfigBucket, node+"."+module+"."+key, fmt.Sprintf(`{"node":%q,"module":%q,"key":%q,"value":%s,`+
`"source":"default"}`, node, module, key, d.Default))
}
}
}
@@ -158,6 +158,7 @@ func TestNatsTheEnvironmentOfTheCataloguesModulesIsWhatTheOldFormComposed(t *tes
if !manifest.Moved || len(manifest.Items) == 0 {
t.Fatalf("%s is not a moved contributor of items", m)
}
b.defaults(t, m, read(m+"/module.json"))
texts := map[string]string{}
var planned []PlannedItem
for _, it := range manifest.Items {
@@ -133,7 +133,11 @@ func serve(ctx context.Context, credential, socket, stateDir string) error {
if err != nil {
return err
}
client := &Client{Node: c.Node, JS: js, Conn: nc, Secrets: secrets,
up, err := watchSocket(ctx, socket)
if err != nil {
return err
}
client := &Client{Node: c.Node, SocketUp: up, JS: js, Conn: nc, Secrets: secrets,
Fill: SocketFiller{Socket: socket, Busy: time.Second, Tries: 10},
Templates: &templateStore{dir: filepath.Join(stateDir, "templates"), fetch: HTTPFetch}}
wait := time.Second

Some files were not shown because too many files have changed in this diff Show More