Compare commits

..
Author SHA1 Message Date
mesh-tools cd01f1c196 Merge pull request 'A group's check names every member at its head, also one that publishes no package yet (issue 1921)' (#159) from fix/1921-zero-pins-own-packages into main 2026-11-05 23:00:43 +00:00
jschoubben 261f8d54f7 Fold the unpublished-member test into the group member's pins test (test budget, issue 1921)
mesh/check being carried to 261f8d54f7ff on cd01f1c196d8 | from c0e5bc2834fe, mesh-catalog df8168f4843f
mesh/delivery merged: source.main moved to novox/mesh-delivery@cd01f1c196d8
mesh/delivery-group group fix/1921-zero-pins-own-packages merged: every member is merged
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 23:44:55 +01:00
jschoubben 643ec7cd91 A check names a group member that publishes no package yet at its head by its root module path (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 23:35:23 +01:00
mesh-tools c0e5bc2834 Merge pull request 'The grants part keeps the consumers a module has on another module's stream (issue 1921)' (#158) from fix/1921-heard-consumers into main 2026-11-05 21:28:43 +00:00
jschoubben 6f048a239c The grants part keeps the consumers a module has on another module's stream: created, and only ever gaining filters (issue 1921)
mesh/check being carried to 6f048a239ca3 on c0e5bc2834fe | from fab533dfd0cd
mesh/delivery merged: source.main moved to novox/mesh-delivery@c0e5bc2834fe
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 22:15:47 +01:00
mesh-tools fab533dfd0 Merge pull request 'A group try hears its events: the grants part adds a manifest's new event types, and no open tries is an answer (issue 1921)' (#157) from fix/1921-try-live into main 2026-11-05 21:04:27 +00:00
jschoubben 39285c95bd Rows for ADR 0640 §5: additions change only subjects and filters, on a real bus too (issue 1921)
mesh/check being carried to 39285c95bdae on fab533dfd0cd | from 8c67adaf9136
mesh/delivery merged: source.main moved to novox/mesh-delivery@fab533dfd0cd
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:55:03 +01:00
jschoubben 1b262125ee Open tries test folded; the consumer update granted for every composed consumer; a standing refusal of an addition said once (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:32:04 +01:00
jschoubben bfe4caf196 Texts the mesh shows say a held stream and consumer only ever gain (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:23:16 +01:00
jschoubben 3742a7c54c The description of grants_assert says a held stream gains the subjects a manifest adds (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:22:50 +01:00
jschoubben f1e6520037 The grants part adds a manifest's new events to a held stream and its consumers' filters, only ever adding (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:21:52 +01:00
jschoubben 4ddf78395d None open is an answer: the checker lists open tries as [], the indexer takes null as none (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 21:10:42 +01:00
mesh-tools 8c67adaf91 Merge pull request 'The indexer learns its own build right at its start, and says when a start takes nothing in (issue 1921)' (#156) from fix/1921-own-build-at-start into main 2026-11-05 19:22:04 +00:00
jschoubben e3735da928 The indexer takes the engine's want as its build only while the apply is in progress (issue 1921)
mesh/check being carried to e3735da928f4 on 8c67adaf9136 | from abe9ea3796ea
mesh/delivery merged: source.main moved to novox/mesh-delivery@8c67adaf9136
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 19:53:05 +01:00
jschoubben 78d57e1cc6 A transient forge failure is asked again; only a directory missing from the tree is cached per head (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 19:49:27 +01:00
jschoubben 9f9e743b2f The indexer started during its replacement fixes the successor's build; a start that decides nothing says so; a failing head is said once (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 19:47:42 +01:00
mesh-tools abe9ea3796 Merge pull request 'The running indexer accepts its successor, and a delivery group can be tried whole (issue 1966, issue 1921)' (#155) from fix/1921-group-try into main 2026-11-05 18:34:52 +00:00
jschoubben f9578cb7b4 A try's end is said again until the stream acknowledges it; a waiting try says why; a hand recovery never overlaps a try event (issue 1921, ADR 0639)
mesh/delivery merged: source.main moved to novox/mesh-delivery@abe9ea3796ea
mesh/check being carried to f9578cb7b418 on 8c67adaf9136 | from abe9ea3796ea
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 19:24:45 +01:00
jschoubben 69a2790099 The indexer fixes its own build once at its start and never learns it later; a merge held for an unknown own build is said pending with why and since (issue 1921, ADR 0633 §1)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:47:31 +01:00
jschoubben 88a47be251 A try says its end with the state it ended in, kept in the try's own state, so an end after a checker restart is still said; only the checker publishes try events (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:47:31 +01:00
jschoubben bd5eb33e9e A failed recovery of the open tries is said and retried; every open try is listed; a silent node ends a try as error (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:38:11 +01:00
jschoubben c2b8ac3370 The indexer hears the checker's events after it lists the open tries; rows for the start order and the config skip (issue 1921)
Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:35:22 +01:00
jschoubben 92c38981ea mesh-sdk's tests read mesh-contracts and mesh-delivery beside it (issue 1921)
mesh-sdk compares its bootstrapped seat definitions with their contracts files, read beside it; the checker
places both, at the group's head for a member, else at the trunk, for a check and for a group try alike.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:30:35 +01:00
jschoubben 45386212da A group try's candidate follows the checker's try events, judged in one locked pass (issue 1921)
The checker says try.started (members at their commits) and try.ended for a whole try; the indexer writes and
deletes the candidate from them and finds the open tries again at its start through checker_tries, never reading
the checker's state. The try's digests are judged inside the one pass that takes the seats; the definitions of every
contracts module of each member are read; the config declarations leave a try's candidate alone.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:28:52 +01:00
jschoubben b97e277d2d The indexer writes a group try's candidate from the checker's try and its previews (issue 1921)
A whole try has no pull request, yet compose-candidate and the manifest validator read the indexer's
candidate of a group. The indexer reads the checker's tries, writes candidates.<try id> (number 0 per member) from the
previews, judges its manifests with the seat definitions the members carry, leaves try-* alone in the open message,
deletes it when the try ends, and has a tool that reads and one that drops it.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:13:29 +01:00
jschoubben 8823cc29d6 Merge branch 'fix/validate-successor-with-its-definitions' into fix/1921-group-try 2026-11-05 18:06:35 +01:00
jschoubben 55936c1cd7 Try a delivery group: members at exact commits, judged by the code of a group's check, every failure named (issue 1921)
A try names several repositories, each the exact commit asked. A whole try (not narrowed to a level or
tests) runs the group check's own path: the manifest validator, the module graph and the composition on every
placed node, reading the indexer's candidate of the try's group. A try with no such candidate is an error that
says so, never a pass. A try's candidate is no check: the checker leaves it alone and owes the forge nothing.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 18:05:24 +01:00
jschoubben 31ff708f88 Judge a successor indexer's manifest with the seat definitions its revision carries, not the built-in list (issue 1966, ADR 0633 §1, §6)
The take-in's dry run reads the contracts modules' definitions of the revision's merges and puts them over the seats while it judges; cmd/mesh-judge puts the request's seats over its own. A refusal of the named indexer's manifest is said on take-in.pending and in indexer_pending.

Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i
2026-11-05 17:52:53 +01:00
mesh-tools 2fcd61baa6 Merge pull request 'Take-in holds a merge by reference, so a mesh-catalog merge over the state value cap is taken in (issue 1585, ADR 0633)' (#154) from fix/1585-pending-by-reference into main 2026-11-05 10:48:57 +00:00
jschoubben 655791b8a3 Bridge tool test: the merged-message rows are a subtest of the existing tools test, within the test budget (issue 1585)
mesh/check being carried to 655791b8a351 on 2fcd61baa6eb | from 4596fbe1b7ac
mesh/delivery merged: source.main moved to novox/mesh-delivery@2fcd61baa6eb
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 11:42:10 +01:00
jschoubben 3c832d1a2d Indexer: a held merge's message is asked of the forge bridge, not read from its stream; covered merges leave their candidate; indexer_take_in_merge (issue 1585)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 11:37:13 +01:00
jschoubben 0605962960 Forge bridge: bridge_merged_message answers a merge's merged message again, built as the bridge builds the one it publishes (issue 1585)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 11:34:04 +01:00
jschoubben 343321f11e Indexer: Arrive keeps its message in memory, only the newest merge of a repository is read, an unreadable one is said and covered by a newer one (issue 1585)
The take-in unblocks merges without the read grant on the forge bridge's stream: the message Arrive was handed is used,
the stream is read only for a pending merge not held in memory, and older merges of a repository in a revision are
covered by its newest and never read.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 11:24:18 +01:00
jschoubben 4b7a31b2c7 Indexer: a pending merge keeps a reference to its FORGE message, not a copy, so a merge over the state value cap is held (issue 1585)
The pending value holds the stream sequence and the small fields the waits read; the take-in reads the message from the
forge bridge's stream by sequence. A message the stream no longer holds is said (take-in.pending lost, indexer_pending)
and never taken in silently. indexer_take_in_forge takes one forge message in by its sequence.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 11:21:42 +01:00
mesh-tools 4596fbe1b7 Merge pull request 'Take-in by the indexer a revision names; a group's merges taken in together (issue 1585, ADR 0633)' (#153) from feat/take-in into main 2026-11-05 00:33:44 +00:00
jschoubben c78941e80b Take-in events are the indexer's ordinary events: no stream of its own, which the manifest validator refuses (issue 1585, ADR 0633 §5)
mesh/delivery merged: source.main moved to novox/mesh-delivery@4596fbe1b7ac
mesh/delivery-group group feat/take-in merged: every member is merged
mesh/check being carried to c78941e80b59 on 4596fbe1b7ac | from ae31b66a9e33, mesh-catalog fd4931a75261 | novox/mesh-delivery#153 in group feat/take-in
The group's check failed at 7b7096f3: the group's own manifest validator refuses the indexer's stream "take-in", outside
the bus's predefined forms. The lead's ruling: the indexer says take-in.pending and take-in.taken as its ordinary events
(its emits, mesh.mod.<module>.event.take-in.<kind>), as manifest-accepted is said, and keeps no stream. The wait's key is
in the body only: an emitted event names one event (no wildcard), and a key such as g.<group> is no event name. The
watchman consumes the two events by name on the shared events stream; it needs no replay, since it seeds its waits from
indexer_pending at its start. indexer_pending says the last event of each wait as take-in.<kind>.

Run locally, as the check runs it: cmd/mesh-judge over every manifest of mesh-catalog, mesh-delivery, mesh-host,
mesh-tools, mesh-networking and mesh-media-catalog, with this branch's and mesh-catalog's feat/take-in manifests in the
change case and origin/main's in the trunk case: 171 of 171 accepted in both. The same run with the indexer's manifest at
7b7096f3 refuses it with the check's words.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 00:59:27 +01:00
jschoubben 7b7096f37e Take-in, review round 1: no claim left behind, no sleep at the start, waits seeded at a restart, ends asked readable (issue 1585, ADR 0633)
mesh/delivery-group group feat/take-in rejected: composed together: the checker's verdict at the members' heads (38623984d6ad): mesh-build-go
mesh/delivery closed: a newer head of the same pull request
mesh/check failed (2): mesh-build-go: the indexer's judge at the group's mesh-delivery head 7b7096f37ea76e1…; and 1 more | 7b7096f37ea7 on ae31b66a9e33
- A take-in releases its claims (compare-and-set, only while the claim is still its own) on a lost claim, on a claim it
  cannot read back, and on a merge it fails to take in: a claim left behind refused every later take-in until a
  restart. T5 row with a store that fails once, both ways.
- The start step waits for nothing: with the build it runs unknown it says so and the forge consumer starts; a merged
  message left unacknowledged asks the build again (clause 1). T10 row: the start answers at once and takes nothing in.
- A toolchain report evaluates pending again (ToolchainReported), since the build a revision names moves with the
  toolchain. T7 row: the revision is taken in once the report makes it name the running build.
- The group's end is told by its whole subject (process.ModuleStreamSubject), so a repository named forge/group is
  never taken for one; row in the indexer's handle test.
- The watchman asks the indexer's waits through indexer_pending at its start (invokes), before its durable consumer,
  which does not hear again what it acknowledged before a restart; the comment that said the whole stream is heard is
  corrected. W2 row: a wait seeded within its bound is raised past it.
- bridge_groups names each group's last end ask, its who, why and when, and the bridge's answer (state end-asks).
- The check's words for the indexer's rules: the trunk's manifest validator.
- The bridge's end drops the merged-state test its merge commit already implies.
- Test budget: groupask_test folded into groups_test, and the K1 mesh/check test into the owed status one: 804 unit
  tests (was 806).

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 00:28:56 +01:00
jschoubben 4e4601a1eb Take-in and forge contracts live with their owners: the indexer's takein package, the forge bridge's forgemsg package; mesh-sdk pinned at main (issue 1585, ADR 0633)
The operator, 2026-11-04: logic one module owns lives inside that module as a package of its own, so the take-in step
changes no mesh-sdk. TakeIn, its keys and event names, PendingWait and PendingAnswer move to
modules/mesh-delivery-indexer/takein (its merges a takein.Merge of its own); Forge whole, with its group fields,
ForgeGroupEnd and GroupMerge, moves to modules/mesh-delivery-forge-bridge/forgemsg. Both are nested Go modules that their
readers take by replace, as buildrun is. Every importer moves in this commit: the indexer, the bridge, the watchman and
the view. Forge's private checks (the repository name, the version) are copied small; mesh-sdk main still holds an unused
Forge until its own fold removes it.

The indexer, bridge, watchman, view and checker pin mesh-sdk 5a38a12 (origin/main), no feat/take-in commit.

The bridge's integration bus now grants the state end-asks that bridge_group_end_again added: without it every test of
the package failed on a permissions violation. Test budget: two new test functions (TestForge, TestTakeInEvents) and the
two the join added are paid by merging four pairs in internal/bridge; the repository stays at 806 unit tests.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-05 00:05:01 +01:00
jschoubben a395b49be2 WIP (ADR 0633 declined): watchman pinned to the group's mesh-sdk head; the other pins, the gate's const swap and the full checks not done
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:24:26 +01:00
jschoubben 1040c8ec46 Forge bridge: the process takes the keys of performed from internal/groups, one copy (issue 1585)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:24:05 +01:00
jschoubben 92dc5f03d0 Merge commit 'ef13e63' into feat/take-in 2026-11-04 23:23:38 +01:00
jschoubben ef13e63268 indexer: TestIndexerWhole runs the take-in step whole, a restart and a group (ADR 0633, issue 1585 I8a, I8b)
A revision changing the indexer writes only what the named build needs and is
taken in when that build starts; a gathered group stays across the restart and
its end makes one take-in.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:22:35 +01:00
jschoubben 9c6e70e555 indexer: before the pending state is on the bus, take a merge in at once and start without asking the own build (ADR 0633, issue 1585)
TestIndexerWhole found the start waiting for ever on a build no node-engine
answered, so the forge consumer never started, though nothing can be pending.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:19:07 +01:00
jschoubben ad9a353484 Forge bridge: bridge_groups and bridge_group_end_again, the end said again through the state end-asks (issue 1585, ADR 0633 §4)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:17:27 +01:00
jschoubben a762701e3d forge bridge: a member's merged message names its group, and the group's end is said once (ADR 0633 §4, issue 1585)
The group is read from the gate's merges state the bridge already reads;
the end comes only once every approved member is merged and its merged
message written, never on a refusal, with a message id the stream's
duplicate window holds to one.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:14:44 +01:00
jschoubben d6b6f24e98 forge bridge: a member's merged message names its group, and the group's end is said once (ADR 0633 §4, issue 1585)
The group is read from the gate's merges state the bridge already reads;
the end comes only once every approved member is merged and its merged
message written, never on a refusal, with a message id the stream's
duplicate window holds to one.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:11:53 +01:00
jschoubben ed4f07b0d8 Watchman: a take-in wait past its bound heard in the replay is said once the consumer has caught up, not after a time (issue 1585, ADR 0633 §5)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:10:23 +01:00
jschoubben 7609f2d9a2 Watchman: a replayed pending that its taken ends is never raised; a drop then a pending again is covered (issue 1585, ADR 0633 §5)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:09:20 +01:00
jschoubben d11a5af113 View: view_tiers names a module the indexer holds back, read through indexer_pending (issue 1585, ADR 0633 §5)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:09:20 +01:00
jschoubben 531d05ab4b indexer: hear a group's end, keep merges pending, take in at start (ADR 0633 §2, §4, issue 1585)
The consumer tells a group's end apart by its name before the forge decode,
a merged message goes through the pending state, and the pending revisions
naming this build are taken in before the consumer starts; take-in events go
on the indexer's own stream. Until the pending state is on the bus every
merge is taken in at once, as before.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:08:37 +01:00
jschoubben 071a28f218 indexer: a source revision is taken in only by the indexer it names (ADR 0633 §1-§5, issue 1585)
A merged message is kept in the indexer's own state pending and the merges
there are computed together on overlays of the indexer's states: taken in
whole when the indexer's fingerprint is the running build, otherwise only
the indexer's own entry, graph key and verdict are written and the rest
waits for the named indexer. A group's members are gathered until its end.
Waits are said on the stream take-in, read by indexer_pending and steered
by indexer_take_in_pending and indexer_drop_pending.

Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 23:07:02 +01:00
jschoubben 72633722c5 Checker test: the accepted-indexer row asserts the trunk's judge said so (issue 1585, T22)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 22:55:27 +01:00
jschoubben 0579e4e74d Watchman: hear the indexer's take-in stream and raise take-in-waits past 60 minutes (revision) or 30 (group) (issue 1585, ADR 0633 §5)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 22:54:21 +01:00
jschoubben 2cb3198049 Checker: judge the new indexer's manifest with the trunk's judge, and fail a held verdict (issue 1585, ADR 0633 clause 6)
Claude-Session: https://claude.ai/code/session_01JG73KG7AfSdh6FP8VrBqCn
2026-11-04 22:47:19 +01:00
145 changed files with 8944 additions and 954 deletions
@@ -62,8 +62,6 @@ type fakeBuildModules struct {
js jetstream.JetStream
ctx context.Context
failBuild atomic.Bool
// failRetakeable makes every build fail as a lookup that timed out (delivery.Retake), which passes by itself.
failRetakeable atomic.Bool
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
refusePush atomic.Bool
mu sync.Mutex
@@ -119,8 +117,6 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any,
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
}
return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil
case f.failRetakeable.Load() && tool != "build_image_copy":
return nil, fmt.Errorf("%slookup git.novox.be on 10.10.0.3:53: i/o timeout", delivery.Retake)
case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs
return nil, fmt.Errorf("go build of %v: boom", args["artifact"])
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
@@ -495,47 +491,30 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
// failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
// the key again as a new revision of the same key, now done.
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
t.Run("a new modules value with the same fingerprint", func(t *testing.T) {
b := startBed(t)
b.fake.failBuild.Store(true)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
}
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
failed, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
b.fake.failBuild.Store(false)
// The indexer writes the module again (here at a later commit), with the same fingerprint.
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
b.await(t, "ca-trust", isState(delivery.BuildDone))
done, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
if done.Revision() <= failed.Revision() {
t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
}
})
// 1748/AC6: a build failed retakeable is built again after its backoff, with no new modules value or other event, and
// ends done; the key says it was retaken.
t.Run("a retakeable failure after its backoff", func(t *testing.T) {
b := startBed(t)
b.fake.failRetakeable.Store(true)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", map[string]string{"MESH_BUILDER_RETAKE_BACKOFF": "1s"})
if v := b.await(t, "ca-trust", isState(delivery.BuildFailed)); !delivery.Retakeable(v.Error) {
t.Fatalf("build = %+v, want failed retakeable", v)
}
b.fake.failRetakeable.Store(false) // the lookup passes
if v := b.await(t, "ca-trust", isState(delivery.BuildDone)); v.Retakes < 1 {
t.Errorf("done key = %+v, want its retake counted", v)
}
})
b := startBed(t)
b.fake.failBuild.Store(true)
b.archive(t, "ca-trust", "modules/ca-trust", commit)
b.start(t, "n1", nil)
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
}
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
failed, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
b.fake.failBuild.Store(false)
// The indexer writes the module again (here at a later commit), with the same fingerprint.
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
b.await(t, "ca-trust", isState(delivery.BuildDone))
done, err := b.builds.Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
if done.Revision() <= failed.Revision() {
t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
}
}
// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a
@@ -77,12 +77,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
}
}
var retakeBackoff time.Duration // 0 is build.DefaultRetakeBackoff
if v := strings.TrimSpace(getenv("MESH_BUILDER_RETAKE_BACKOFF")); v != "" {
if retakeBackoff, err = time.ParseDuration(v); err != nil {
return fmt.Errorf("MESH_BUILDER_RETAKE_BACKOFF: %w", err)
}
}
tools := &buildrun.ToolchainSet{}
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
@@ -127,8 +121,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Logf: logger.Printf,
Stored: storedVia(calls.Call),
Node: cred.Node,
// A key failed retakeable (delivery.Retake) is taken again by itself after this wait, at most delivery.MaxRetakes times (issue 1748).
RetakeBackoff: retakeBackoff,
}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4 h1:K2niszrlnJ7JMiXI6xg4gG90YCd/tqNLGBvujLInI48=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -66,10 +66,6 @@ type ClaimOptions struct {
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
Stored func(ctx context.Context, references []string) (gone []string, err error)
// Retaking is this claim being the builder's backoff taking a retakeable failed key again (Builder.armRetake, issue
// 1748): only it adds to BuilderBuild.Retakes, so the cap of delivery.MaxRetakes counts the backoff's own retakes and
// not a look another event makes while the backoff waits.
Retaking bool
}
// StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store
@@ -157,8 +153,7 @@ func Unreferenced(b delivery.BuilderBuild) bool {
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
// which wait for another fingerprint (a retakeable failure, delivery.Retakeable, is taken again too; the new
// revision keeps Retakes, and counts one more only when it is the backoff's claim, ClaimOptions.Retaking), and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
@@ -217,10 +212,6 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
return nil, nil
case cur.State == delivery.BuildFailed:
building.Retakes = cur.Retakes // the count rides on the key's value, so it survives a restart (issue 1748)
if opt.Retaking && delivery.Retakeable(cur.Error) {
building.Retakes++
}
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
case cur.State == delivery.BuildQueued && cur.Waits != "":
if opt.Published == nil || !opt.Published(cur.Waits) {
@@ -1,179 +0,0 @@
package build
import (
"context"
"fmt"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// clock is the builder's fake clock for the retake backoff (issue 1748): After records what is asked and fire runs it.
type clock struct {
mu sync.Mutex
waits []time.Duration
pending []func()
}
func (c *clock) after(d time.Duration, f func()) func() {
c.mu.Lock()
defer c.mu.Unlock()
c.waits, c.pending = append(c.waits, d), append(c.pending, f)
return func() {}
}
// fire runs every timer set so far, once, and answers how many there were.
func (c *clock) fire() int {
c.mu.Lock()
fs := c.pending
c.pending = nil
c.mu.Unlock()
for _, f := range fs {
f()
}
return len(fs)
}
func retakeRig(t *testing.T) (*rig, *clock) {
r := newRig(t, goBundle)
c := &clock{}
r.b.RetakeBackoff, r.b.After = 10*time.Second, c.after
return r, c
}
const lookupFailure = delivery.Retake + "go mod download: lookup git.novox.be on 10.10.0.3:53: i/o timeout"
// 1748: a key failed retakeable is taken again by the builder's backoff, and only by it counts toward the cap.
func TestTheBuilderRetakesARetakeableFailure(t *testing.T) {
// AC3: claimed again as a new revision after the backoff, with no other event; the count kept on the key stops it after
// delivery.MaxRetakes, and a builder that starts again reads the count.
t.Run("after its backoff, at most three times", func(t *testing.T) {
r, c := retakeRig(t)
r.bm.fails["ace/build_go_build"] = lookupFailure
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes != 0 {
t.Fatalf("build = %+v, want failed retakeable with no retake yet", v)
}
if len(c.waits) != 1 || c.waits[0] != 10*time.Second {
t.Fatalf("backoffs = %v, want one of 10s", c.waits)
}
rev := r.st.revs[r.key]
for retake := 1; retake <= delivery.MaxRetakes; retake++ {
if c.fire() != 1 {
t.Fatalf("retake %d: no timer was set", retake)
}
r.b.Wait()
v = valueOf(t, r.st, r.key)
if v.Retakes != retake || r.st.revs[r.key] <= rev {
t.Fatalf("retake %d: key = %+v at revision %d (was %d)", retake, v, r.st.revs[r.key], rev)
}
rev = r.st.revs[r.key]
}
if len(c.pending) != 0 || len(c.waits) != delivery.MaxRetakes || c.waits[1] != 20*time.Second || c.waits[2] != 40*time.Second {
t.Fatalf("after %d retakes: %d pending, backoffs %v; want none pending and 10s, 20s, 40s", delivery.MaxRetakes, len(c.pending), c.waits)
}
if n := len(r.bm.callsOf("build_go_build")); n != 1+delivery.MaxRetakes {
t.Errorf("%d builds, want the first and %d retakes", n, delivery.MaxRetakes)
}
// A builder that starts again reads the count from the key: at the cap it arms nothing, below it the next wait.
restarted := func(retakes int) *clock {
rr, cc := retakeRig(t)
rr.st, rr.b.Claimer.Store = r.st, r.st
data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildFailed, Error: lookupFailure, Retakes: retakes})
if err != nil {
t.Fatal(err)
}
r.st.vals[r.key] = data
rr.b.init()
rr.b.known["m"] = delivery.Module{FP2: fp1, Manifest: mdigest('a')}
if err := rr.b.Sweep(context.Background()); err != nil {
t.Fatal(err)
}
return cc
}
if cc := restarted(delivery.MaxRetakes); len(cc.waits) != 0 {
t.Errorf("a key retaken %d times was armed again: %v", delivery.MaxRetakes, cc.waits)
}
if cc := restarted(1); len(cc.waits) != 1 || cc.waits[0] != 20*time.Second {
t.Errorf("a key retaken once was armed with %v, want one wait of 20s", cc.waits)
}
})
// AC4: a failure of the source (a compile error, FailedUntilChanged) is never taken again by the backoff.
t.Run("a failure of the source is never taken again", func(t *testing.T) {
for _, reason := range []string{"go build of tool: undefined: x", delivery.NotPinned + "up: names no digest", delivery.VersionTaken + "@x/y@1 holds another file"} {
r, c := retakeRig(t)
r.bm.fails["ace/build_go_build"] = reason
v := r.consider(t, "modules/m")
if v.State != delivery.BuildFailed || delivery.Retakeable(v.Error) {
t.Fatalf("%q: build = %+v, want failed and not retakeable", reason, v)
}
if len(c.waits) != 0 || c.fire() != 0 {
t.Errorf("%q: the backoff was armed: %v", reason, c.waits)
}
}
})
// Only the backoff's claim counts: a look another event makes while the backoff waits (a toolchain change, a new modules
// value) takes the key again and leaves the count as it was, so those looks cannot spend the retakes inside the pause.
t.Run("another event's look leaves the count", func(t *testing.T) {
r, c := retakeRig(t)
r.bm.fails["ace/build_go_build"] = lookupFailure
r.consider(t, "modules/m")
for i := 0; i < 2*delivery.MaxRetakes; i++ {
r.b.Reconsider(context.Background())
r.b.Wait()
}
if v := valueOf(t, r.st, r.key); v.Retakes != 0 || v.State != delivery.BuildFailed {
t.Fatalf("after Reconsider looks the key = %+v, want failed with Retakes 0", v)
}
if c.fire() != 1 {
t.Fatal("the backoff's own wait was not armed once")
}
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.Retakes != 1 {
t.Errorf("after the backoff's claim the key = %+v, want Retakes 1", v)
}
})
// The journal says each retake with its count and the log's entry of the build's end names it.
t.Run("the journal and the log say the count", func(t *testing.T) {
r, c := retakeRig(t)
var said []string
r.b.Logf = func(f string, a ...any) { said = append(said, strings.TrimSpace(fmt.Sprintf(f, a...))) }
var ended []delivery.BuildLogEntry
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) {
if e.Event == delivery.BuildEndedEvent {
ended = append(ended, e)
}
}
r.bm.fails["ace/build_go_build"] = lookupFailure
r.consider(t, "modules/m")
c.fire()
r.b.Wait()
if len(ended) != 2 || ended[0].Retakes != 0 || ended[1].Retakes != 1 {
t.Fatalf("ended entries = %+v, want retakes 0 then 1; journal: %v", ended, said)
}
if joined := strings.Join(said, "\n"); !strings.Contains(joined, "taken again after its backoff (retake 1 of 3)") {
t.Errorf("the journal says %q, want the retake with its count", joined)
}
})
// The key's error opens with the retake prefix only when every failing lane failed retakeable.
t.Run("the key error opens with the prefix only when every lane is retakeable", func(t *testing.T) {
for _, c := range []struct {
name string
errs []string
prefix bool
}{
{"one retakeable lane", []string{"mesh-build-go on ace: the step failed: " + lookupFailure}, true},
{"two retakeable lanes", []string{"a: " + lookupFailure, "b: " + lookupFailure}, true},
{"a retakeable and a compile error", []string{"a: " + lookupFailure, "b: go build of tool: undefined: x"}, false},
{"a compile error", []string{"b: go build of tool: undefined: x"}, false},
} {
got := keyError(c.errs)
if has := strings.HasPrefix(got, delivery.Retake); has != c.prefix || delivery.Retakeable(got) != c.prefix {
t.Errorf("%s: keyError = %q, want prefix %v", c.name, got, c.prefix)
}
}
})
}
@@ -79,12 +79,6 @@ type Builder struct {
// time.Now. Node is the node the builder runs on, which asked the build.
Now func() time.Time
Node string
// RetakeBackoff is the wait before the builder takes a key that failed retakeable (delivery.Retakeable) again by itself,
// doubled with each retake (issue 1748): DefaultRetakeBackoff when 0. After runs f once d has passed and returns a
// stop; nil waits a context's bound (ADR 0437 §8: the builder holds no timer). The wait is re-armed from the key's value at
// start (Sweep): a build's work is not a message of the builder's own on a stream, so there is no delivery delay to ride.
RetakeBackoff time.Duration
After func(d time.Duration, f func()) (stop func())
once sync.Once
sem chan struct{}
@@ -108,14 +102,8 @@ type Builder struct {
waiting map[string]map[string]chan string
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
hands map[string]*handWatch
// armed are the keys with a retake wait running, by build key; retaking are the keys whose backoff has passed and whose
// next claim is the backoff's own (ClaimOptions.Retaking).
armed, retaking map[string]bool
}
// DefaultRetakeBackoff is the first wait before a retakeable failure is taken again; each retake doubles it.
const DefaultRetakeBackoff = 60 * time.Second
type planned struct {
plan Plan
err error
@@ -132,7 +120,6 @@ func (b *Builder) init() {
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands = map[string]*handWatch{}
b.armed, b.retaking = map[string]bool{}, map[string]bool{}
})
}
@@ -200,14 +187,6 @@ func (b *Builder) Sweep(ctx context.Context) error {
continue
}
newly = b.notePublished(cur) || newly
if current[key] && cur.State == delivery.BuildFailed {
if module, _, err := delivery.ParseBuilderBuildKey(key); err == nil {
b.mu.Lock()
fp := b.known[module].FP2
b.mu.Unlock()
b.armRetake(ctx, key, module, fp, cur) // a restarted builder reads the count and goes on
}
}
if current[key] || cur.State != delivery.BuildBuilding {
continue
}
@@ -450,11 +429,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
trunk := b.trunkFP[key]
first := !b.tried[key]
b.tried[key] = true
retaking := b.retaking[key]
delete(b.retaking, key)
b.mu.Unlock()
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
Stored: b.storedOnce(key), Retaking: retaking})
Stored: b.storedOnce(key)})
if err != nil {
b.logf("%s: %v", module, err)
return
@@ -500,7 +477,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
b.logf("%s: left building, the toolchains moved under it", cl.Key)
return
}
result.By, result.Manifest, result.Retakes = b.Claimer.By, m.Manifest, cl.Build.Retakes
result.By, result.Manifest = b.Claimer.By, m.Manifest
if len(cl.Gone) > 0 && result.State == delivery.BuildDone {
result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
}
@@ -516,7 +493,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
if result.State == delivery.BuildFailed {
b.logf("%s: failed: %s", cl.Key, result.Error)
b.armRetake(ctx, cl.Key, module, m.FP2, result)
}
if result.State == delivery.BuildQueued {
b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits)
@@ -527,57 +503,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
}
}
// armRetake takes a key that failed retakeable again by itself once its backoff has passed, as a new revision (issue 1748):
// only an error that opens with delivery.Retake, and at most delivery.MaxRetakes times, counted on the key's value. A key
// failed for its source (a compile error, FailedUntilChanged) is never armed.
func (b *Builder) armRetake(ctx context.Context, key, module, fp string, v delivery.BuilderBuild) {
if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes >= delivery.MaxRetakes || ctx.Err() != nil {
return
}
b.mu.Lock()
if b.armed[key] {
b.mu.Unlock()
return
}
b.armed[key] = true
b.mu.Unlock()
wait := b.RetakeBackoff
if wait <= 0 {
wait = DefaultRetakeBackoff
}
wait <<= v.Retakes
b.logf("%s: failed retakeable; taken again in %s (retake %d of %d)", key, wait, v.Retakes+1, delivery.MaxRetakes)
after := b.After
if after == nil {
// The builder holds no timer (ADR 0437 §8: a wait is a context's bound): the wait is the bound of a context, which
// the builder's own context ends early.
after = func(d time.Duration, f func()) func() {
wait, stop := context.WithTimeout(ctx, d)
go func() {
<-wait.Done()
if errors.Is(wait.Err(), context.DeadlineExceeded) && ctx.Err() == nil {
f()
}
}()
return stop
}
}
after(wait, func() {
b.mu.Lock()
delete(b.armed, key)
m, known := b.known[module]
b.mu.Unlock()
if ctx.Err() != nil || !known || m.FP2 != fp {
return // a newer fingerprint supersedes it: that one is built by its own event
}
b.logf("%s: taken again after its backoff (retake %d of %d)", key, v.Retakes+1, delivery.MaxRetakes)
b.mu.Lock()
b.retaking[key] = true
b.mu.Unlock()
b.consider(ctx, module, m)
})
}
// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
func (b *Builder) stale(m delivery.Module, plan Plan) string {
@@ -799,19 +724,6 @@ func keyError(errs []string) string {
}
}
}
// A retakeable failure opens the key's error only when every failing lane is one (issue 1748): a lane that failed on its
// source would be built again for nothing.
all := len(errs) > 0
for _, e := range errs {
all = all && strings.Contains(e, delivery.Retake)
}
if all {
stripped := make([]string, len(errs))
for i, e := range errs {
stripped[i] = strings.Replace(e, delivery.Retake, "", 1)
}
return delivery.Retake + strings.Join(stripped, "; ")
}
return strings.Join(errs, "; ")
}
@@ -1043,7 +955,6 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
e.Against = append([]string(nil), m.Inputs2.Bases...)
}
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
e.Retakes = result.Retakes
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
return e
}
@@ -212,25 +212,4 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
if files < 1 {
t.Fatalf("read %d source files, want the program's", files)
}
// The one bounded wait the builder holds for its own sake is the retake backoff (issue 1748): a goroutine that waits a
// context's deadline and then acts. It lives in Builder.armRetake only; any other source holding that idiom is refused.
const idiom = "context.DeadlineExceeded"
var holders []string
for _, root := range []string{"cmd", "internal"} {
filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return err
}
src, _ := os.ReadFile(path)
for _, fn := range strings.Split(string(src), "\nfunc ") {
if strings.Contains(fn, idiom) && !strings.HasPrefix(fn, "(b *Builder) armRetake(") {
holders = append(holders, path)
}
}
return nil
})
}
if len(holders) != 0 {
t.Errorf("%v waits for a context's deadline and acts: the retake backoff in Builder.armRetake is the one allowed bounded wait (issue 1748); any other wait is a timer", holders)
}
}
@@ -103,15 +103,25 @@ func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{
{Name: check.TryTool,
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"},
Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " +
"name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " +
"or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " +
"each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " +
"the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " +
"the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " +
"check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " +
"Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " +
"(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " +
"is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto",
"items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}},
"repository": map[string]any{"type": "string", "description": "<owner>/<name>, for a try of one member (instead of members)"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"},
"tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"},
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}},
Run: func(args map[string]any) (any, error) {
@@ -126,16 +136,32 @@ func TryTools(tr Trying) []stdio.Tool {
if tr.Now != nil {
now = tr.Now
}
var members []check.TryAskMember
if raw, ok := args["members"]; ok {
list, isList := raw.([]any)
if !isList {
return nil, fmt.Errorf("members is a list of {repository, head, trunk}")
}
for _, item := range list {
m, isMap := item.(map[string]any)
if !isMap {
return nil, fmt.Errorf("a member is {repository, head, trunk}")
}
members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")})
}
}
id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"),
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now())
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now())
if err != nil {
return nil, err
}
return map[string]any{"id": id}, nil
}},
{Name: check.TryReadTool,
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " +
"when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " +
"every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " +
"the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " +
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
@@ -146,7 +172,7 @@ func TryTools(tr Trying) []stdio.Tool {
return check.ReadTry(tr.Tries, str(args, "id"))
}},
{Name: check.TryEndTool,
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
@@ -170,6 +196,24 @@ func TryTools(tr Trying) []stdio.Tool {
}
}
// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is
// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries
// whose start event it missed, instead of reading the checker's state.
func OpenTriesTools(tr Trying) []stdio.Tool {
return []stdio.Tool{{Name: check.TriesTool,
Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " +
"trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " +
"missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
if tr.Tries == nil {
return nil, fmt.Errorf("the tries state cannot be read here")
}
open, err := check.OpenTries(tr.Tries)
return map[string]any{"tries": open}, err
}}}
}
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
const Seat = "mesh-delivery-checker"
@@ -325,7 +369,7 @@ func main() {
_, err := verdicts.Update(key, json.RawMessage(value), revision)
return err
}}
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
if err := stdio.Serve("", logged(tools)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
@@ -8,6 +8,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"mesh-delivery-checker/internal/check"
)
@@ -106,6 +108,39 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("the verb tried a repository nothing checks")
}
// A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the
// description says what it does in the words a search for "try a group" finds.
later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}})
if err != nil || g.(map[string]any)["id"] == id {
t.Errorf("a group try = %v, %v, want a try of its own", g, err)
}
if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}},
"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("members and a repository were both taken")
}
if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil {
t.Error("members that are no list were taken")
}
for _, tl := range tools {
for _, words := range []string{"group"} {
if !strings.Contains(strings.ToLower(tl.Description), words) {
t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description)
}
}
}
if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") ||
!strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") {
t.Errorf("the try's description does not say what a group try does: %q", d)
}
if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") {
t.Errorf("open tries tool = %+v", open)
} else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 {
t.Errorf("open tries = %v, %v, want the two tries asked", out, err)
}
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
@@ -862,9 +862,11 @@ func (s triesStore) Create(key string, value []byte) error {
// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
// verdict and no status owed (ADR 0501 §5).
// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the
// key's watch, and writes the merge previews the indexer makes it from (issue 1921).
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
b := startBed(t)
asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is
b.start(t, nil)
store := triesStore{b.buckets[check.TriesBucket], b.ctx}
id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
@@ -872,19 +874,36 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Fatal(err)
}
var last check.Try
// The indexer's candidate of the try's group (here written by the test), at the heads of its members.
raw0, _ := store.Get(id)
first, _ := check.DecodeTry(raw0)
cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash,
Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}}
cand.FPSet = delivery.FingerprintSet(cand.Pulls)
data, err := delivery.EncodeCandidate(cand)
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey(id)
if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(id)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
t.Fatalf("try = %+v, want passed on the trunk", last)
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil {
t.Fatalf("try = %+v, want passed on the trunk with its composition made", last)
}
if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") {
t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.<heads>", got, id)
}
if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
t.Errorf("the try fetched nothing")
}
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} {
if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
t.Errorf("a try wrote %s: %v", name, keys)
}
@@ -29,7 +29,10 @@ type judgeDeps struct {
trunks *check.Trunks
kept check.KV
previews check.KV
repos *repoModules
// candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no
// manifest or composition judged, said.
candidates check.TryCandidates
repos *repoModules
// published is the indexer's packages state as it is followed (packages.go).
published *publishedPackages
forge check.Forge
@@ -157,7 +160,7 @@ func newJudge(d judgeDeps) check.Judge {
nodes = d.nodes
}
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
if d.nodes != nil && d.builds != nil && d.repos != nil {
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
}
@@ -21,6 +21,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -189,7 +190,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Verdicts: verdicts.all,
}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf,
// A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group.
Say: func(ctx context.Context, event string, body tryevent.Event) error {
data, _ := json.Marshal(body)
_, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second))
return err
}}
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -216,7 +223,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]},
repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
@@ -333,6 +340,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
each := func(e jetstream.KeyValueEntry) {
if check.IsTryGroup(e.Key()) {
return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921)
}
if opOf(e) {
ch.Candidate(ctx, e.Key(), e.Value())
} else {
@@ -0,0 +1,72 @@
package main
import (
"context"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge
// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says
// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there.
const candidateWaitsOnPreview = "no merge preview is computed"
// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the
// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the
// key's own watch, with no timer between reads, and gives up at the bound.
type tryCandidates struct{ kv jetstream.KeyValue }
func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) {
key, err := delivery.CandidatesKey(group)
if err != nil {
return delivery.Candidate{}, err
}
if bound <= 0 {
bound = time.Minute
}
waiting, cancel := context.WithTimeout(ctx, bound)
defer cancel()
w, err := t.kv.Watch(waiting, key) // the value now, then each change
if err != nil {
return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err)
}
defer func() { _ = w.Stop() }()
last := "the indexer holds none"
for {
select {
case <-waiting.Done():
return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last)
case e := <-w.Updates():
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
c, err := delivery.DecodeCandidate(e.Value())
switch {
case err != nil:
last = "what it holds is not a candidate: " + err.Error()
case c.HeadsHash != heads:
last = "it holds heads " + c.HeadsHash
case waitsOnPreview(c):
last = "it has not yet the merge previews' fingerprints"
default:
return c, nil
}
}
}
}
// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read.
func waitsOnPreview(c delivery.Candidate) bool {
for _, p := range c.Pulls {
for _, m := range p.Fps {
if m.None2 == candidateWaitsOnPreview {
return true
}
}
}
return false
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -29,18 +29,23 @@ const (
nodeEngineModule = "mesh-host"
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
networkingRepository = "novox/mesh-networking"
// contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its
// tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the
// trunk (a group try lays them out the same way: besideFor serves the check and the try).
contractsRepository = "novox/mesh-contracts"
deliveryRepository = DeliveryRepository
)
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
// composition moved into them (ADR 0593).
var besideRules = map[string][]string{
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository},
catalogueRepository: {SDKRepository, nodeEngineRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
@@ -55,6 +55,8 @@ type Case struct {
Watch func(askID string, step *delivery.Step)
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check.
Wait func(why string)
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
Kind string
@@ -1035,8 +1035,25 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
// mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6).
cycle string
trunkCycle bool
// indexer makes web the holder of the seat mesh-catalog, and the trunk's manifest validator answers for it: "held" with no verdict,
// "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the
// trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23)
indexer string
ownJudge bool
// try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's
// group (issue 1921): the same judge, the same verdicts.
try bool
}{
{name: "accepted"},
{name: "accepted in a group try", try: true},
{name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`},
{name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
{name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"},
{name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"},
{name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"},
{name: "the indexer refused by the trunk's manifest validator", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"},
{name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the manifest validator does not know its fields new_field, so it waits"},
{name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"},
{name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"},
{name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps",
fails: "refuses the manifest of mail: mail reads a state web no longer keeps"},
@@ -1077,6 +1094,10 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
}
b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed}
}
if row.indexer != "" && row.indexer != "heldplain" {
b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web"},
web: {Module: "web", Claims: []catalogue.Claim{{Name: CatalogSeat}}}}
}
var judged []delivery.ComposeJob
answer := b.asks.Answer
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
@@ -1085,6 +1106,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
}
judged = append(judged, *a.Compose)
read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused}
switch row.indexer {
case "held", "heldplain":
read = composer.ManifestRead{}
case "aside":
read.SetAside = "new_field"
}
verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase,
Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}},
{Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}})
@@ -1092,9 +1119,29 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result}
}
p := pullOf(repoA, 7, head1)
if row.ownJudge {
p = pullOf(DeliveryRepository, 7, head1)
b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"}
}
p.Modules = []string{"web"}
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
v := b.judge(t, caseOf(t, p)).Verdict
cs := caseOf(t, p)
if row.try {
cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil)
b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}}
}
v := b.judge(t, cs).Verdict
if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool {
return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator")
}) {
t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's manifest validator", row.name, cs.notes)
}
if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) {
t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged)
}
if row.ownJudge {
judged = judged[1:]
}
if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk ||
judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir {
t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v)
@@ -30,7 +30,7 @@ type ManifestReader interface {
// refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest,
// when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it.
func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.j.Read == nil || st.c.Try != nil {
if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() {
return nil
}
trunk := st.j.Manifests.ManifestDigests()
@@ -139,8 +139,18 @@ type Examiner struct {
Nodes interface {
Names(ctx context.Context) ([]string, error)
}
Now func() time.Time
Logf func(format string, args ...any)
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
// and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no
// manifest or composition judged, said.
Candidates TryCandidates
Now func() time.Time
Logf func(format string, args ...any)
}
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
// bound; an error says it did not come (and why).
type TryCandidates interface {
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error)
}
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
@@ -246,6 +256,50 @@ type state struct {
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
unpreviewed []string
// noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and
// it was not composed; empty when it has one, and for a pull request's check.
noCandidate string
}
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds
// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews.
// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run).
func (st *state) takeTryCandidate(ctx context.Context) {
if st.c.Try == nil || !st.c.Try.Whole() {
return
}
no := func(why string) {
st.noCandidate = "the group's manifests and composition were not judged: " + why
st.c.note(st.noCandidate)
}
if st.j.Candidates == nil {
no("the checker reads no candidate of the indexer's")
return
}
if len(st.unpreviewed) > 0 {
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
return
}
if st.c.Wait != nil {
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
defer st.c.Wait("")
}
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
if err != nil {
no(err.Error())
return
}
for i, p := range st.c.Pulls {
for _, cp := range cand.Pulls {
if cp.Repository == p.Repository && cp.Head.Head == p.Head {
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
}
}
}
st.c.FPSet = cand.FPSet
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
@@ -256,6 +310,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
st.takeTryCandidate(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
@@ -282,7 +337,14 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil {
if st.narrowTry() {
return
}
if st.noCandidate != "" {
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate}
return
}
if st.j.Compositions == nil {
return
}
if len(st.unpreviewed) > 0 {
@@ -433,7 +495,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
switch {
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
case len(merged) == len(modules):
st.j.writePreview(ctx, p, merged)
default:
@@ -562,7 +624,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
}
if len(t.fetched.Conflicts) > 0 {
if st.c.Try == nil {
if !st.narrowTry() {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
@@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request,
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
// cannot be asked or does not answer fails the check, said, never passes it.
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.c.Try != nil {
if st.j.Manifests == nil || st.narrowTry() {
return nil
}
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
@@ -120,26 +120,12 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
req.Predefined = list
st.c.note("the manifests are judged against the predefined keys of " + from)
}
raw, err := json.Marshal(req)
if err != nil {
return fail(goModule, err.Error())
cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at)
if v != nil {
return v
}
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Compose: &job}
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
return fail(goModule, "the judge of the change's manifests: "+err.Error())
}
cv, trunkCase, why := judgeAnswer(step)
if why != "" {
return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
if v := st.refuseUnmovableIndexer(ctx, req, touched, cv, at); v != nil {
return v
}
// A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a
// warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case
@@ -147,7 +133,13 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
var refused, warned []string
for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) {
r := cv.Manifests[module]
if r.Accepted || r.Refused == "" {
if r.Accepted {
continue
}
if r.Refused == "" { // held: a verdict not given is not one given (ADR 0633 clause 6)
if slices.Contains(touched, module) {
refused = append(refused, fmt.Sprintf("the indexer's judge at %s holds the manifest of %s: it gave no verdict", at, module))
}
continue
}
had := map[string]bool{}
@@ -183,6 +175,97 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
return nil
}
// runJudge asks the Go build seat to build the judge at the commit and run it over the request, and reads its change and
// trunk cases; a failed verdict when it cannot be asked or does not answer.
func (st *state) runJudge(ctx context.Context, req composer.Request, touched []string, commit, at string) (change, trunk composer.CaseVerdict, v *delivery.Verdict) {
raw, err := json.Marshal(req)
if err != nil {
return change, trunk, fail(goModule, err.Error())
}
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Compose: &job}
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return change, trunk, fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
return change, trunk, fail(goModule, "the judge of the change's manifests: "+err.Error())
}
change, trunk, why := judgeAnswer(step)
if why != "" {
return change, trunk, fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
}
return change, trunk, nil
}
// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and
// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4).
const CatalogSeat = "mesh-catalog"
// refuseUnmovableIndexer fails the check when the trunk's manifest validator, the one the running parts hold, would
// refuse or hold the change's new indexer manifest: such an indexer would never run, and every revision behind it would
// wait for ever (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog.
// When the change judges with its own head's manifest validator (it changes it), the manifest is judged by the trunk's
// as well. The verdicts read are the trunk's manifest validator's: its change case, where refused, held (not accepted
// with no reason) or with a field set aside (a field the running manifest validator does not know waits, ADR 0601 §3)
// each fail it, naming the module and the problems.
func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict {
if st.j.Read == nil {
return nil
}
var indexer string
for _, module := range touched {
m, err := st.j.Read.Manifest(ctx, req.Cases[0].Manifests[module].Digest)
if err != nil {
return fail(goModule, fmt.Sprintf("the manifest of %s cannot be read, so the indexer cannot be ruled out: %v", module, err))
}
for _, c := range m.Claims {
if c.Name == CatalogSeat {
indexer = module
}
}
}
if indexer == "" {
return nil
}
if st.judgeHead != "" {
trunk := ""
if st.j.Trunks != nil {
trunk = st.j.Trunks.Heads()[DeliveryRepository]
}
if trunk == "" {
return fail(goModule, "the new indexer cannot be judged by the trunk's manifest validator: the trunk head of "+DeliveryRepository+" is not known")
}
var v *delivery.Verdict
judgedAt = "mesh-delivery's trunk " + trunk
if cv, _, v = st.runJudge(ctx, req, touched, trunk, judgedAt); v != nil {
return v
}
}
r, ok := cv.Manifests[indexer]
var problems []string
switch {
case !ok:
problems = append(problems, "the manifest validator gave no verdict")
case r.Refused != "":
problems = append(problems, strings.Split(r.Refused, "\n")...)
case !r.Accepted:
problems = append(problems, "the manifest validator holds it, with no verdict yet")
case r.SetAside != "":
problems = append(problems, "the manifest validator does not know its fields "+r.SetAside+", so it waits")
}
if len(problems) > 0 {
return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's manifest validator at %s does not accept its manifest: %s",
indexer, CatalogSeat, judgedAt, strings.Join(problems, "; ")))
}
st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's manifest validator at %s", indexer, judgedAt))
return nil
}
// judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot.
func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) {
if step.State != delivery.StepPassed {
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"maps"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -58,6 +59,28 @@ func (st *state) packagesOf(repository string) delivery.PackageVersions {
delete(versions, pkg)
}
}
// A package of a member that no indexer state publishes yet (a repository whose packages are all new) is in no list above,
// and the build would look for it as the trunk's latest version, which does not exist (issue 1921). The member's root
// module path stands for its repository: the build module takes the head of a repository from any package of it.
for _, m := range st.c.Pulls {
if delivery.SameRepository(m.Repository, repository) || len(m.Head) != 40 {
continue
}
name := strings.TrimPrefix(m.Repository, "novox/")
if m.Repository == name {
continue
}
held := false
for pkg := range versions {
if repo, _, ok := delivery.PackageRepository(pkg); ok && repo == m.Repository {
held = true
break
}
}
if !held {
versions[delivery.ForgeModulePrefix+name] = m.Head
}
}
if len(versions) == 0 {
return nil
}
@@ -60,6 +60,15 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
if got := packagesOfFetch(t, b, "novox/mesh-catalog"); len(got) != 2 || got[SDKModulePath] != trunkS || got[contract] != trunkA {
t.Errorf("a pull request alone is built against %v, want every package at its latest published version", got)
}
// A member that publishes no package yet is still named, by its root module path, at its head (issue 1921).
b = newBench(t)
b.ex.Published = published{}
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf("novox/mesh-host", 8, head2))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("an unpublished member: verdict = %+v", v)
}
if got := packagesOfFetch(t, b, repoA); len(got) != 1 || got["git.novox.be/novox/mesh-host"] != head2 {
t.Errorf("the member's packages = %v, want the unpublished member's root at its head %s alone", got, head2)
}
}
type runsAt map[string]string
@@ -111,10 +120,16 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
}
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
const contracts = "novox/mesh-contracts"
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2})
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
}
// mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and
// mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way).
if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 {
t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2)
}
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
}
@@ -164,6 +164,7 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
if err != nil {
return VerdictRead{}, err
}
keys = slices.DeleteFunc(keys, IsTryGroup)
sort.Strings(keys)
keys, cut := capKeys(keys, "")
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
@@ -444,6 +445,7 @@ func (r Reader) Status() (Status, error) {
if err != nil {
return st, err
}
keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check
sort.Strings(keys)
total := len(keys)
keys, cut := capKeys(keys, "")
@@ -14,6 +14,7 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -27,6 +28,8 @@ const (
TryTool = "checker_try"
TryReadTool = "checker_try_read"
TryEndTool = "checker_try_end"
// TriesTool lists the tries that wait or run (OpenTries).
TriesTool = "checker_tries"
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
TriesState = "tries"
TriesBucket = delivery.CheckerModule + "_" + TriesState
@@ -62,18 +65,28 @@ var (
// Try is the value of a tries key.
type Try struct {
V int `json:"v"`
ID string `json:"id"`
V int `json:"v"`
ID string `json:"id"`
// Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository
// names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a
// group try, issue 1921). A try written before members read as one member.
Repository string `json:"repository"`
// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
Members []TryMember `json:"members,omitempty"`
// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
// level runs. Empty is every test of both levels.
Tests string `json:"tests,omitempty"`
Level string `json:"level,omitempty"`
State string `json:"state"`
// Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the
// try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts.
Told bool `json:"told,omitempty"`
// Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing;
// try-read gives it, so a try that sits still says why.
Waiting string `json:"waiting,omitempty"`
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration.
@@ -83,6 +96,11 @@ type Try struct {
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
// Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to
// some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept
// verdicts taken): the same as a group's pull-request check reports.
Composition *delivery.Composition `json:"composition,omitempty"`
Notes []string `json:"notes,omitempty"`
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
Asks []string `json:"asks,omitempty"`
@@ -94,6 +112,23 @@ type Try struct {
Ended time.Time `json:"ended,omitzero"`
}
// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is
// merged onto. Branch is the trunk's branch when the trunk was asked by name.
type TryMember struct {
Repository string `json:"repository"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
}
// MembersOf are the try's members: its Members, or the one its headline names.
func (t Try) MembersOf() []TryMember {
if len(t.Members) > 0 {
return t.Members
}
return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}}
}
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
const TryAsked = "asked"
@@ -148,6 +183,17 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) {
}
}
// waiting writes what the try waits for now ("" when it waits for nothing).
func (w *tryWriter) waiting(why string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.stopped || w.t.Waiting == why {
return
}
w.t.Waiting = why
w.write()
}
// write writes the record at the revision written last; a refusal stops the writes.
func (w *tryWriter) write() {
data, err := encodeTry(w.t)
@@ -195,6 +241,7 @@ func DecodeTry(raw []byte) (Try, error) {
if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
}
t.Members = t.MembersOf()
return t, nil
}
@@ -206,6 +253,14 @@ func encodeTry(t Try) ([]byte, error) {
// TryAsk is what checker_try is given.
type TryAsk struct {
Repository, Head, Trunk, Tests, Level string
// Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the
// trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused.
Members []TryAskMember
}
// TryAskMember is one repository of a group try as it is asked.
type TryAskMember struct {
Repository, Head, Trunk string
}
// TryStore is the tries state as the tools see it.
@@ -221,21 +276,16 @@ func NotChecked(repository string) error {
return fmt.Errorf("no level of %s is checked by the checker", repository)
}
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit
// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks.
func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
head := strings.ToLower(strings.TrimSpace(a.Head))
switch {
case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
return TryMember{}, fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
case !tryHeadOK.MatchString(head):
return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo)
}
trunkHead := ""
for r, h := range trunks {
@@ -244,20 +294,67 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
}
}
if trunkHead == "" {
return "", NotChecked(repo)
return TryMember{}, NotChecked(repo)
}
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
m := TryMember{Repository: repo, Head: head}
switch trunk := strings.TrimSpace(a.Trunk); {
case trunk == "":
t.Trunk = trunkHead
m.Trunk = trunkHead
case tryHeadOK.MatchString(strings.ToLower(trunk)):
t.Trunk = strings.ToLower(trunk)
m.Trunk = strings.ToLower(trunk)
case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
t.Trunk, t.Branch = trunkHead, trunk
m.Trunk, m.Branch = trunkHead, trunk
default:
return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo)
}
return m, nil
}
// sameMembers says whether two tries ask the same commits onto the same trunk commits.
func sameMembers(a, b []TryMember) bool {
return slices.EqualFunc(a, b, func(x, y TryMember) bool {
return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk
})
}
// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of
// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing
// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no
// repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
asked := a.Members
if a.Repository != "" || a.Head != "" || a.Trunk != "" {
if len(asked) > 0 {
return "", errors.New("name the members, or one repository with its head, not both")
}
asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}}
}
if len(asked) == 0 {
return "", errors.New("name what to try: members (repository and head each), or a repository and its head")
}
switch {
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
}
var members []TryMember
for _, m := range asked {
tm, err := tryMember(trunks, m)
if err != nil {
return "", err
}
if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) {
return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository)
}
members = append(members, tm)
}
slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) })
first := members[0]
t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members,
Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
keys, err := store.Keys()
if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err)
@@ -291,10 +388,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if !live[i] {
continue
}
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level {
return cur.ID, nil
}
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State)
if cur.HoldsANode() {
holding = append(holding, said)
} else {
@@ -379,12 +476,50 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
return t, nil
}
// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the
// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes
// no verdict and owes the forge no status for it.
func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) }
// TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct {
Level string
Run string
}
// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by
// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor
// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921).
func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" }
// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number
// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by.
func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case {
cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
var heads []delivery.Head
for _, m := range t.MembersOf() {
cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
cs.Trunk[m.Repository] = m.Trunk
heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head})
}
cs.HeadsHash, _ = delivery.HeadsHash(heads)
cs.Heads = cs.HeadsHash
return cs
}
// compositionMade says whether a composition was made; notMadeWords what it says when it was not.
func compositionMade(c *delivery.Composition) bool {
return c != nil && c.State != delivery.CompositionNotRun
}
func notMadeWords(c *delivery.Composition) string {
if c == nil || c.Words == "" {
return "no reason was given"
}
return c.Words
}
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
type Tryer struct {
@@ -396,7 +531,13 @@ type Tryer struct {
Now func() time.Time
Logf func(format string, args ...any)
// Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The
// indexer, which hears them, writes and deletes the candidate of a whole try's group.
// It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again.
Say func(ctx context.Context, event string, body tryevent.Event) error
mu sync.Mutex
ending map[string]bool // tries whose end is being said now
wg sync.WaitGroup
inflight map[string]context.CancelFunc
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
@@ -436,6 +577,7 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
stop()
}
tr.mu.Unlock()
tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate
return
}
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
@@ -491,6 +633,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
t.State, t.Started = TryRunning, tr.now().UTC()
t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on
data, err := encodeTry(t)
if err != nil {
return
@@ -502,12 +645,15 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
cs := &Case{Key: key, Group: key, Watch: progress.heard,
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
Trunk: map[string]string{t.Repository: t.Trunk},
Queued: t.Asked,
Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
if t.Told {
started := tryevent.Event{ID: key}
for _, m := range t.MembersOf() {
started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
tr.say(ctx, tryevent.Started, started)
}
cs := TryCase(key, t, progress.heard)
cs.Wait = progress.waiting
judging := ctx
if tr.NotTaken > 0 {
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
@@ -517,12 +663,18 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
}
t, rev = progress.end()
t.Waiting = ""
v := out.Verdict
t.Ended = tr.now().UTC()
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
t.Composition = v.Composition
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
for _, n := range cs.Notes() {
if len(cs.Pulls) > 1 {
t.Said = sayMembers(t.MembersOf()) + ": " + t.Said
}
t.Notes = cs.Notes()
for _, n := range t.Notes {
t.Said += " | " + n
}
switch {
@@ -532,6 +684,16 @@ func (tr *Tryer) run(ctx context.Context, key string) {
if IsLostAsk(v) || IsNotTaken(v) {
t.Failed = []string{v.Failed.Words}
}
case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer):
// A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it
// did not, naming the node (ADR 0637 §3).
t.State = TryError
t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words}
case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition):
// A whole try that judged the tests and not the manifests and the composition has not tried what its group's check
// judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure.
t.State = TryError
t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)}
case v.State == delivery.VerdictPass:
t.State = TryPass
default:
@@ -549,6 +711,97 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
tr.logf("%s: %s", key, t.State)
tr.sendEnd(ctx, key)
}
// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's
// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and
// by hand.
func (tr *Tryer) sendEnd(ctx context.Context, key string) {
tr.mu.Lock()
if tr.ending[key] {
tr.mu.Unlock()
return
}
if tr.ending == nil {
tr.ending = map[string]bool{}
}
tr.ending[key] = true
tr.mu.Unlock()
defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }()
raw, rev, found, err := tr.Tries.Get(ctx, key)
if err != nil || !found {
return
}
t, err := DecodeTry(raw)
if err != nil || !t.Over() || !t.Told {
return
}
if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil {
return // not acknowledged: Told stays, and the next Put or Reconsider says the end again
}
t.Told = false
data, err := encodeTry(t)
if err == nil {
_, err = tr.Tries.Update(ctx, key, data, rev)
}
if err != nil && !errors.Is(err, ErrConflict) {
tr.logf("%s: its end was said and Told could not be cleared: %v", key, err)
}
}
// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start).
func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error {
if tr.Say == nil {
return nil
}
err := tr.Say(ctx, event, body)
if err != nil {
tr.logf("%s of %s could not be said: %v", event, body.ID, err)
}
return err
}
// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at
// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in
// parallel.
func OpenTries(store TryStore) ([]tryevent.Open, error) {
keys, err := store.Keys()
if err != nil {
return nil, fmt.Errorf("the tries cannot be listed: %w", err)
}
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) != len(keys[j]) {
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
found := make([]*tryevent.Open, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
if err != nil || len(raw) == 0 {
return nil
}
t, err := DecodeTry(raw)
if err != nil || t.Over() {
return nil
}
o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()}
for _, m := range t.MembersOf() {
o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
found[i] = &o
return nil
}); err != nil {
return nil, err
}
out := []tryevent.Open{} // none open is an empty list, never null: a reader tells it from no answer (issue 1921)
for _, o := range found {
if o != nil {
out = append(out, *o)
}
}
return out, nil
}
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
@@ -579,6 +832,15 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
return out, true
}
// sayMembers names a try's members, each repository at its short commit.
func sayMembers(ms []TryMember) string {
var out []string
for _, m := range ms {
out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head))
}
return strings.Join(out, " + ")
}
// orNone joins the tries said, or says none.
func orNone(said []string) string {
if len(said) == 0 {
@@ -3,11 +3,14 @@ package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -70,6 +73,25 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
}
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
// A group try is the same try whatever the order of its members; the commits asked are the commits tried.
group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}}
gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second))
if err != nil || gid == id {
t.Fatalf("the group try = %q, %v, want a try of its own", gid, err)
}
group.Members[0], group.Members[1] = group.Members[1], group.Members[0]
if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid {
t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid)
}
if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS {
t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members)
}
if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 {
t.Errorf("open tries = %+v, %v, want the two queued", open, err)
}
if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") {
t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid)
}
onNode := func(i int) Try {
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
@@ -112,59 +134,154 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr
}
}
for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"},
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} {
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"},
// A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked.
{},
{Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} {
if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil {
t.Errorf("%+v was taken", bad)
}
}
}
func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5
// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked.
type fakeTryCandidates struct {
cand delivery.Candidate
err error
asked []string
// during is called while the try waits for the candidate.
during func()
}
func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) {
if f.during != nil {
f.during()
}
f.asked = append(f.asked, group+"."+heads)
return f.cand, f.err
}
// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the
// indexer's candidate of the try's group.
func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) {
comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}}
cands := &fakeTryCandidates{cand: cand}
b.ex.Compositions, b.ex.Candidates = comps, cands
return comps, cands
}
// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request
// check; the two cannot drift, and a later check of the same heads keeps what the try passed.
func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921
ctx := context.Background()
kv := newMemKV()
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now())
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now())
if err != nil {
t.Fatal(err)
}
b := newBench(t)
comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"})
written := &writtenTries{memKV: kv}
tr := &Tryer{Tries: written, Judge: b.ex}
var waiting string
cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for
var said []string
tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State))
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
// The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate).
if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass {
t.Errorf("events = %v, want the start with 2 members and the end", said)
}
// What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's.
for _, row := range []struct {
name, event, body string
ok bool
}{
{"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true},
{"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true},
{"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false},
{"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false},
{"a body that is no JSON", tryevent.Ended, `try-7`, false},
} {
if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) {
t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok)
}
}
got := readTry(t, kv, id)
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
if !strings.Contains(waiting, "candidate") || got.Waiting != "" {
t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting)
}
if got.Told { // the end was said: nothing is owed
t.Errorf("the ended try still owes its end: %+v", got)
}
if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 {
t.Errorf("open tries after the end = %v, %v", open, err)
} else if raw, _ := json.Marshal(open); string(raw) != "[]" { // issue 1921: none open is [] on the wire, never null
t.Errorf("none open is answered %s, want []", raw)
}
if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 ||
got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests
t.Fatalf("try = %+v", got)
}
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
// keeps each ask's node and state.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
written.running[2].Steps[0].Node != "shanks" {
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 {
t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition)
}
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
t.Errorf("the ended try's steps = %+v", got.Steps)
// The same key the indexer's candidate of the group has: the try's id and the heads of its members.
if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") {
t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id)
}
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
if len(got.Asks) != 1 {
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
// Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked {
t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running)
}
if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 {
t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks)
}
for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage)
if _, err := delivery.AskStepKey("shanks", a); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", a, err)
}
}
// Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref.
fetch := b.steps.tool("fetch")
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
if len(fetch) != 2 {
t.Fatalf("fetch = %+v, want one per member", fetch)
}
if keys, _ := b.prev.Keys(ctx); len(keys) != 0 {
t.Errorf("a try wrote the previews %v", keys)
for _, f := range fetch {
want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)]
if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" {
t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1])
}
}
// A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve.
if keys, _ := b.prev.Keys(ctx); len(keys) != 2 {
t.Errorf("a whole try wrote the previews %v, want one per member", keys)
}
// An ended try is not run again.
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if n := len(b.steps.tool("fetch")); n != 1 {
if n := len(b.steps.tool("fetch")); n != 2 {
t.Errorf("an ended try was fetched again: %d fetches", n)
}
// The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of
// the same head on the same trunk runs none of it again (and a push of other code runs only what changed).
before := len(b.steps.tool("test"))
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 ||
out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels)
}
if after := len(b.steps.tool("test")); after != before {
t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before)
}
}
func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
@@ -175,10 +292,16 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error {
t.Error("a try of one level and some tests said an event: it has no candidate")
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if open, _ := OpenTries(memTries{kv}); len(open) != 0 {
t.Errorf("open = %v", open)
}
tests := b.steps.tool("test")
if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" {
t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests)
@@ -192,6 +315,24 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") {
t.Errorf("try = %+v", got)
}
t.Run("a try of one level keeps only that level", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now())
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
keys, _ := b.kept.Keys(ctx)
if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 {
t.Fatalf("try = %+v, kept %v", got, keys)
}
for _, k := range keys {
if !strings.HasPrefix(k, delivery.LevelUnit+".") {
t.Errorf("a try of the unit level kept %s", k)
}
}
})
}
func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) {
@@ -208,26 +349,135 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
t.Errorf("try = %+v", got)
}
// A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside
// the failed tests (the group's check does as much), and a candidate of other heads is not the try's.
for _, row := range []struct {
name string
cand *fakeTryCandidates
comp delivery.CompositionVerdict
state, said string
}{
{name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"},
{name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}},
state: TryError, said: "g14: " + delivery.ComposeNoAnswer},
{name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"},
state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"},
} {
kv := newMemKV()
b := newBench(t)
comps, _ := wholeBench(b, delivery.Candidate{})
b.ex.Candidates, comps.v = row.cand, row.comp
if row.comp.State == "" {
comps.v.State = delivery.CompositionPass
}
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
var ends []string
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
ends = append(ends, e.State)
}
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail
t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state)
}
if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) {
t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said)
}
}
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
}
func TestATryLeftRunningIsTakenOver(t *testing.T) {
ctx := context.Background()
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State = TryRunning
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
tr.Reconsider(ctx)
tr.Wait()
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
// Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did
// not start, whatever way the try ended.
for _, row := range []struct {
name string
told bool
failFirst bool // the stream does not acknowledge the first end said
after func(t *testing.T, kv *memKV, id string)
ends []string
}{
{name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}},
{name: "left running after its start was said", told: true, ends: []string{TryPass}},
{name: "left running with no start said", told: false, ends: []string{TryPass}},
{name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) {
if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil {
t.Fatal(err)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State, cur.Told = TryRunning, row.told
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
if row.after != nil {
row.after(t, kv, id)
}
b := newBench(t)
wholeBench(b, delivery.Candidate{})
var ends []string
failed := false
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
if row.failFirst && !failed {
failed = true
return errors.New("the stream did not acknowledge")
}
ends = append(ends, e.State)
}
return nil
}}
tr.Reconsider(ctx)
tr.Wait()
if row.failFirst && !readTry(t, kv, id).Told {
t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again")
}
tr.Reconsider(ctx) // said once: the second look finds nothing owed
tr.Wait()
if row.after == nil {
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
}
}
if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told {
t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends)
}
})
}
// An ended try is kept TryKeep and removed by the next Reconsider after.
t.Run("an ended try is kept seven days", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
ended := time.Now()
cur.State, cur.Ended = TryPass, ended
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
now := ended.Add(TryKeep - time.Hour)
tr := &Tryer{Tries: kv, Now: func() time.Time { return now }}
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); !found {
t.Fatal("the try was removed before TryKeep")
}
now = ended.Add(TryKeep + time.Hour)
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); found {
t.Error("the try was kept past TryKeep")
}
})
}
+29 -3
View File
@@ -130,14 +130,40 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
if m.SecretsOwner != "mesh-delivery-checker" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) {
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
// module of the repository declares a stream carrying a try event (the indexer and the others only consume them).
siblings, err := filepath.Glob("../*/module.json")
if err != nil || len(siblings) < 2 {
t.Fatalf("the sibling manifests = %v, %v", siblings, err)
}
for _, f := range siblings {
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
var other struct {
Module string `json:"module"`
Streams []stream `json:"streams"`
}
if err := json.Unmarshal(raw, &other); err != nil {
t.Fatalf("%s: %v", f, err)
}
for _, st := range other.Streams {
for _, e := range st.Events {
if strings.HasPrefix(e, "try.") && other.Module != m.Module {
t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e)
}
}
}
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
// is asked again by hand (checker_ask_again).
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
+2 -1
View File
@@ -18,7 +18,7 @@
"mesh-build-node.toolchain.reported"
],
"streams": [
{ "name": "checks", "events": ["preview.**", "owed.**"] }
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }
],
"state": [
{ "name": "asks", "history": 1 },
@@ -56,6 +56,7 @@
"checker_previews",
"checker_pull",
"checker_status",
"checker_tries",
"checker_try",
"checker_try_end",
"checker_try_read",
@@ -0,0 +1,266 @@
// Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585): the runtime
// launches it for a call. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends;
// bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the
// bridge is the other program and serves no request subject (ADR 0437 §4).
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"mesh-delivery-forge-bridge/internal/bridge"
"mesh-delivery-forge-bridge/internal/groups"
)
// Tool names.
const (
GroupsTool = "bridge_groups"
EndAgainTool = "bridge_group_end_again"
// MergedTool answers the merged message of a merge on request (issue 1585).
MergedTool = "bridge_merged_message"
)
// asker writes an end-asks value and waits for the bridge's answer to it.
type asker func(key string, value []byte) (groups.EndAsk, error)
// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them.
func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string) (json.RawMessage, error)) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: MergedTool,
Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " +
"pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " +
"and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " +
"group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " +
"build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " +
"with the forge message whole, or why it cannot: the gate holds no merge with that commit, or the forge did not answer.",
Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"},
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}}},
Run: func(args map[string]any) (any, error) {
raw, err := merged(str(args, "repository"), str(args, "merge"))
if err != nil {
return nil, err
}
return map[string]any{"message": raw}, nil
}},
{Name: GroupsTool,
Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " +
"merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " +
"or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " +
"what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " +
"answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"},
"open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}},
Run: func(args map[string]any) (any, error) {
rows, unread, err := groupsRead()
if err != nil {
return nil, err
}
only, _ := args["open"].(bool)
group := str(args, "group")
kept := rows[:0]
for _, r := range rows {
if (group != "" && r.Group != group) || (only && !r.Open) {
continue
}
kept = append(kept, r)
}
return map[string]any{"groups": kept, "unread": unread}, nil
}},
{Name: EndAgainTool,
Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " +
"in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " +
"writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " +
"a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " +
"a member is not merged or its merged message is not written. Answers whether it was written.",
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"},
"why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return map[string]any{"group": a.Group, "written": a.Written, "answer": a.Answer, "by": a.By, "why": a.Why}, nil
}},
}
}
// MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's
// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files
// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them.
func MergedOf(raws map[string][]byte, repository, merge string, trees bridge.Trees, files bridge.Files) (json.RawMessage, error) {
owner, repo, ok := strings.Cut(repository, "/")
if !ok || owner == "" || repo == "" {
return nil, fmt.Errorf("%q is not owner/name", repository)
}
for _, key := range slices.Sorted(maps.Keys(raws)) {
m, err := delivery.DecodeMerge(raws[key])
if err != nil {
continue
}
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
return delivery.SameRepository(p.Repository, repository) && p.Commit == merge
})
if at < 0 {
continue
}
p := m.Pulls[at]
msg, err := bridge.MergedMessage(owner, repo, p.Number, m.Group, p.Head.Head, merge, trees, files)
if err != nil {
return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err)
}
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
return nil, err
}
if _, approved := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 {
f.Group, f.Members = m.Group, groups.MembersOf(m)
}
out, err := forgemsg.EncodeForge(f)
return out, err
}
return nil, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge)
}
// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of
// each in end-asks with the bridge's answer.
func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) {
keys, err := merges.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
}
raws := map[string][]byte{}
for _, k := range keys {
e, err := merges.Get(k)
if err != nil || e == nil {
continue
}
raws[k] = e.Value
}
pkeys, err := performed.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
}
published, ended := map[string]bool{}, map[string]uint64{}
for _, k := range pkeys {
switch {
case strings.HasPrefix(k, "published."):
published[k] = true
case strings.HasPrefix(k, "groups."):
if e, err := performed.Get(k); err == nil && e != nil {
rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64)
ended[strings.TrimPrefix(k, "groups.")] = rev
}
}
}
akeys, err := asks.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the bridge's state %s cannot be listed: %w", groups.EndAsksState, err)
}
askRaws := map[string][]byte{}
for _, k := range akeys {
if e, err := asks.Get(k); err == nil && e != nil {
askRaws[k] = e.Value
}
}
rows, unread := groups.Rows(raws, published, ended, askRaws)
return rows, unread, nil
}
// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer.
func askEnd(s stdio.KeptState, bound time.Duration) asker {
return func(key string, value []byte) (groups.EndAsk, error) {
answered := make(chan groups.EndAsk, 1)
var rev uint64
var mu sync.Mutex
take := func(revision uint64, v []byte) {
mu.Lock()
mine := rev
mu.Unlock()
if mine == 0 || revision <= mine {
return
}
if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() {
select {
case answered <- a:
default:
}
}
}
if err := s.Watch(key, func(c stdio.StateChange) error {
if c.Op == "put" {
take(c.Revision, c.Value)
}
return nil
}); err != nil {
return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
if e, err := s.Get(key); err == nil && e != nil {
take(e.Revision, e.Value)
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case a := <-answered:
return a, nil
case <-wait.Done():
return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key)
}
}
}
// askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it.
func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) {
return stdio.Ask(key, body)
}
func main() {
merges, performed, asks := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState)
tools := Tools(askEnd(asks, 25*time.Second), time.Now,
func() ([]groups.Row, []string, error) { return read(merges, performed, asks) },
func(repository, merge string) (json.RawMessage, error) {
keys, err := merges.Keys()
if err != nil {
return nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
}
raws := map[string][]byte{}
for _, k := range keys {
if e, err := merges.Get(k); err == nil && e != nil {
raws[k] = e.Value
}
}
ctx := context.Background()
return MergedOf(raws, repository, merge, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller))
})
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err)
os.Exit(1)
}
}
@@ -0,0 +1,190 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
"mesh-delivery-forge-bridge/internal/groups"
)
// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is
// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group,
// who asked and why, and answers what the bridge answered.
func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
enc := func(group string, state2 delivery.MergeState, c string) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 1, Head: h1}, State: delivery.MergeMerged, Commit: c1},
{Head: delivery.Head{Repository: "novox/b", Number: 2, Head: h2}, State: state2, Commit: c}}})
if err != nil {
t.Fatal(err)
}
return v
}
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
merges := map[string][]byte{"ended": enc("ended", delivery.MergeMerged, c2), "ready": enc("ready", delivery.MergeMerged, c2),
"gathered": enc("gathered", delivery.MergeMerging, ""), "solo": lone, "bad": []byte("{")}
performed := map[string]bool{}
for _, c := range []string{c1, c2} {
performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true
}
// The state end-asks keeps the last ask of the group ready, answered, and one that does not read.
answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC),
Written: true, Answer: "the end of the group is written again", Answered: time.Date(2026, 11, 4, 22, 0, 1, 0, time.UTC)})
asks := map[string][]byte{"ready": answeredAsk, "gathered": []byte("{")}
var asked groups.EndAsk
var askedKey string
ask := func(key string, value []byte) (groups.EndAsk, error) {
askedKey = key
a, err := groups.DecodeEndAsk(value)
a.Written, a.Answer, a.Answered = true, "the end of the group is written again", time.Now()
asked = a
return a, err
}
tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) },
func() ([]groups.Row, []string, error) {
rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks)
return rows, unread, nil
}, nil)
call := func(name string, args map[string]any) any {
for _, tl := range tools {
if tl.Name == name {
out, err := tl.Run(args)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
return out
}
}
t.Fatalf("no tool %s", name)
return nil
}
rowsOf := func(args map[string]any) map[string]groups.Row {
raw, _ := json.Marshal(call(GroupsTool, args))
var out struct {
Groups []groups.Row `json:"groups"`
Unread []string `json:"unread"`
}
if err := json.Unmarshal(raw, &out); err != nil {
t.Fatal(err)
}
m := map[string]groups.Row{}
for _, r := range out.Groups {
m[r.Group] = r
}
if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" {
t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread)
}
return m
}
all := rowsOf(map[string]any{})
if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 {
t.Fatalf("groups = %+v", all)
}
if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published {
t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r)
}
if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil {
t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk)
}
if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) {
t.Errorf("gathered = %+v", r)
}
if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" {
t.Errorf("open = %+v, want ready and gathered", open)
}
if one := rowsOf(map[string]any{"group": "ready"}); len(one) != 1 {
t.Errorf("group ready = %+v", one)
}
out := call(EndAgainTool, map[string]any{"group": "ready", "why": "the indexer never took it in"}).(map[string]any)
if askedKey != "ready" || asked.Group != "ready" || asked.Why != "the indexer never took it in" || out["written"] != true {
t.Errorf("the ask %q %+v, answered %v", askedKey, asked, out)
}
for _, tl := range tools {
if tl.Name == EndAgainTool {
if _, err := tl.Run(map[string]any{}); err == nil {
t.Error("an ask naming no group was taken")
}
}
}
// 1585: bridge_merged_message answers the merged message as the bridge publishes it: for a lone pull request the same bytes
// the event gives through Convert, for a member of a delivery group those with its group and members set as the process
// sets them; a merge the gate does not hold, or a forge that does not answer, is said.
t.Run("bridge_merged_message answers the merged message as published", func(t *testing.T) {
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
trees := func(owner, repo, sha string) (string, map[string]string, error) {
return "root-" + sha[:4], map[string]string{"": "r", "modules/a": "ta-" + sha[:4]}, nil
}
files := func(owner, repo, path, ref string) ([]byte, error) {
if strings.HasSuffix(path, "module.json") {
return []byte(`{"module":"a"}`), nil
}
return nil, bridge.ErrNotFound
}
published := func(repo string, number int, head, branch, merge string) forgemsg.Forge {
body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge})
msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files)
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
return f
}
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
group, _ := delivery.EncodeMerge(delivery.Merge{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 6, Head: h1}, State: delivery.MergeMerged, Commit: c1},
{Head: delivery.Head{Repository: "novox/b", Number: 7, Head: h2}, State: delivery.MergeMerging}}})
for _, c := range []struct {
name string
raws map[string][]byte
repository string
merge string
want forgemsg.Forge
fail string
}{
{name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)},
{name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge {
f := published("a", 6, h1, "feat/g", c1)
f.Group, f.Members = "feat/g", []delivery.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}}
return f
}()},
{name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"},
} {
t.Run(c.name, func(t *testing.T) {
raw, err := MergedOf(c.raws, c.repository, c.merge, trees, files)
if c.fail != "" {
if err == nil || !strings.Contains(err.Error(), c.fail) {
t.Fatalf("err %v, want %q", err, c.fail)
}
return
}
want, _ := forgemsg.EncodeForge(c.want)
if err != nil || !bytes.Equal(raw, want) {
t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want)
}
})
}
if _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, func(string, string, string) (string, map[string]string, error) {
return "", nil, errors.New("the forge does not answer")
}, files); err == nil || !strings.Contains(err.Error(), "does not answer") {
t.Errorf("a forge that does not answer: %v", err)
}
})
}
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"time"
@@ -66,58 +65,12 @@ func callerOn(nc *nats.Conn, user string) caller {
}
}
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
const commitTreesTool = "gitea.gitea_commit_trees"
// treesVia asks the forge for a commit's trees through call.
func treesVia(ctx context.Context, call caller) bridge.Trees {
return func(owner, repo, sha string) (string, map[string]string, error) {
raw, err := call(ctx, commitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return "", nil, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
}
if a.Root == "" {
return "", nil, fmt.Errorf("the answer of %s names no root tree", commitTreesTool)
}
return a.Root, a.Trees, nil
}
return bridge.TreesVia(ctx, bridge.Caller(call))
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
const getFileTool = "gitea.gitea_get_file"
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
// recognises it as /: 404 /).
var notFound = regexp.MustCompile(`: 404 `)
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
func filesVia(ctx context.Context, call caller) bridge.Files {
return func(owner, repo, path, ref string) ([]byte, error) {
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
if notFound.MatchString(err.Error()) {
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
}
return nil, err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return []byte(*a.File.Content), nil
}
return bridge.FilesVia(ctx, bridge.Caller(call))
}
@@ -9,6 +9,7 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -120,8 +121,8 @@ func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" {
t.Fatalf("published on %v", pub.subjects)
}
m, err := delivery.DecodeForge(pub.data[0])
if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
m, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
t.Errorf("message %+v err %v", m, err)
}
f.refuse = nil
@@ -188,8 +189,8 @@ func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/
if len(f.calls) != 1 || len(pub.data) != 1 {
t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data))
}
m, err := delivery.DecodeForge(pub.data[0])
if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
m, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" {
t.Errorf("message %+v err %v", m, err)
}
@@ -0,0 +1,61 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-forge-bridge/internal/groups"
)
// endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered
// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate
// window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value
// answered already (this bridge's own write) is left.
func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) {
a, err := groups.DecodeEndAsk(value)
if err != nil {
g.logger.Printf("%s.%s holds what is not an end ask, left: %v", groups.EndAsksState, key, err)
return
}
if a.Done() {
return
}
a = g.answerEnd(ctx, key, a, time.Now())
g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer)
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
g.logger.Printf("%s.%s: the answer could not be written: %v", groups.EndAsksState, key, err)
}
}
// readMerges reads one key of the gate's merges state.
func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) {
if g.merges == nil {
return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)")
}
value, rev, found, err := g.merges(ctx, key)
return value, rev, found, err
}
// answerEnd says the group's end again for an ask and returns the ask with its answer.
func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk {
a.Answered = at.UTC()
switch merges, mrev, found, err := g.readMerges(ctx, key); {
case err != nil:
a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error()
case !found:
a.Answer = "the gate holds no merge of the group " + a.Group
case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)):
a.Written, a.Answer = true, "the end of the group is written again"
default:
a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)"
}
return a
}
@@ -0,0 +1,178 @@
package main
import (
"context"
"errors"
"log"
"slices"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each
// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is
// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once.
//
// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.<hash>` for each
// member's merged message written (the hash is the first 16 hex digits of the sha256 of `<repository>#<number>@<merge>`),
// and `groups.<merges key>` once the group's end is written, its value the revision of the gate's merges key it was
// worked out from. A group whose end is not written yet is open (bridge_groups reads them).
// merges reads one key of the gate's merges state.
type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error)
// grouper sets a member's group and says a group's end.
type grouper struct {
merges mergesRead
send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id
done revisions
logger *log.Logger
}
// sendVia writes on FORGE through pub, with the message id.
func sendVia(pub publisher) func(context.Context, string, []byte, string) error {
return func(ctx context.Context, subject string, data []byte, id string) error {
_, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id))
return err
}
}
func kvMerges(kv jetstream.KeyValue) mergesRead {
return func(ctx context.Context, key string) ([]byte, uint64, bool, error) {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, false, nil
}
if err != nil {
return nil, 0, false, err
}
return e.Value(), e.Revision(), true, nil
}
}
// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of
// its approved members and the group has more than one: a lone pull request is no group.
func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (delivery.Merge, uint64, bool) {
if g == nil || g.merges == nil || f.Branch == "" {
return delivery.Merge{}, 0, false
}
key, err := delivery.MergesKey(f.Branch)
if err != nil {
return delivery.Merge{}, 0, false
}
value, rev, found, err := g.merges(ctx, key)
if err != nil {
g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err)
return delivery.Merge{}, 0, false
}
if !found {
return delivery.Merge{}, 0, false
}
m, err := delivery.DecodeMerge(value)
if err != nil {
g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err)
return delivery.Merge{}, 0, false
}
if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 {
return delivery.Merge{}, 0, false
}
return m, rev, true
}
// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a
// merge of no group, is left as it is.
func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) {
f, err := forgemsg.DecodeForge(data)
if err != nil || f.Kind != forgemsg.ForgeMerged {
return data, f, false
}
m, _, ok := g.groupOf(ctx, f)
if !ok {
return data, f, false
}
f.Group, f.Members = m.Group, groupkeys.MembersOf(m)
out, err := forgemsg.EncodeForge(f)
if err != nil {
g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err)
return data, f, false
}
return out, f, true
}
// published keeps that a member's merged message is written, and says the group's end when it was the last.
func (g *grouper) published(ctx context.Context, f forgemsg.Forge) {
if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil {
g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err)
return
}
key, err := delivery.MergesKey(f.Branch)
if err != nil {
return
}
value, rev, found, err := g.merges(ctx, key)
if err != nil || !found {
return
}
g.ended(ctx, key, rev, value)
}
// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its
// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it).
// It is written once per group: `groups.<key>` in performed says it was. again says it is written again by hand
// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it.
func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) {
g.end(ctx, key, rev, value, "")
}
func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool {
m, err := delivery.DecodeMerge(value)
if err != nil || len(m.Approved) < 2 {
return false
}
if again == "" {
if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen {
return false
}
}
end := forgemsg.ForgeGroupEnd{Group: m.Group}
for _, member := range groupkeys.MembersOf(m) {
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number
})
if at < 0 || m.Pulls[at].Commit == "" {
return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered
}
if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok {
return false // its merged message is not written yet: said when it is (published)
}
end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit})
}
// The members in the order the gate merged them, so the last commit names the end.
slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int {
return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) -
slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge })
})
data, err := forgemsg.EncodeForgeGroupEnd(end)
if err != nil {
g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err)
return false
}
id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "")
if again != "" {
id += "." + again
}
if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil {
g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err)
return false
}
if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil {
g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err)
}
g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge)
return true
}
@@ -0,0 +1,141 @@
package main
import (
"bytes"
"context"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// idPub records the subjects, bodies and message ids it is handed.
type idPub struct {
subjects, ids []string
data [][]byte
}
func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error {
p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id)
return nil
}
func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4)
ctx := context.Background()
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
merge := func(states ...delivery.MergePull) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states,
Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}})
if err != nil {
t.Fatal(err)
}
return v
}
sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words}
}
dlv := func(state delivery.MergeState, commit string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit}
}
gate := map[string][]byte{}
var logs bytes.Buffer
pub, done := &idPub{}, memRevisions{}
g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0),
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
v, ok := gate[key]
return v, 7, ok, nil
}}
mergedOf := func(repo string, number int, head, commit string) []byte {
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"})
if err != nil {
t.Fatal(err)
}
return data
}
// A lone pull request's merged message carries no group.
if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" {
t.Errorf("a lone pull request names the group %q", f.Group)
}
// The first member: its message names the group and both members; the group is not ended.
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, ""))
data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1))
if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 {
t.Fatalf("a member's message: %+v %v", back, err)
}
g.published(ctx, f)
// A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later).
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"})
g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"])
// Merged on at the gate before its merged message is written: still no end; then its message ends the group.
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2))
g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"])
if len(pub.subjects) != 0 {
t.Fatalf("an end before every member's merged message is written: %v", pub.subjects)
}
_, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2))
g.published(ctx, f)
g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 {
t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2)
}
end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0])
if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 {
t.Errorf("the end = %+v %v, want both members in their merge order", end, err)
}
if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 {
t.Errorf("performed %v: want groups.feat/take-in kept", done)
}
// Again by hand: written once more, its id not the first's.
if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] {
t.Errorf("the end again by hand: ids %v", pub.ids)
}
// 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on
// the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold.
pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit}
}
askMerge := func(second delivery.MergeState, commit2 string) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2},
Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}})
if err != nil {
t.Fatal(err)
}
return v
}
ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)}
for _, tc := range []struct {
name string
gate map[string][]byte
published bool
written bool
answer string
}{
{"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, true, true, "written again"},
{"a member's message not written", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, false, false, "not written"},
{"a member not merged", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerging, "")}, true, false, "not written"},
{"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"},
} {
pub, done := &idPub{}, memRevisions{}
if tc.published {
done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1
}
g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0),
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
v, ok := tc.gate[key]
return v, 4, ok, nil
}}
got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second))
if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] {
t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids)
}
if tc.written && pub.ids[0] != groupkeys.EndID("feat/take-in", c2, groupkeys.AgainStamp(ask.At)) {
t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0])
}
}
}
@@ -20,6 +20,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -36,7 +37,7 @@ import (
//
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
// user keeps all rights, as it has.
const busConf = `
@@ -83,6 +84,12 @@ accounts {
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
"$KV.mesh-delivery-forge-bridge_performed.>"
"$KV.mesh-delivery-forge-bridge_inputs.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks"
"$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>"
"$KV.mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
@@ -321,13 +328,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(raw.Data)
f, err := forgemsg.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
want := forgemsg.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !delivery.SameForge(f, want) {
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
@@ -416,16 +423,16 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(raw.Data)
f, err := forgemsg.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
want := forgemsg.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
// No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2).
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
if !delivery.SameForge(f, want) {
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
select {
@@ -478,7 +485,7 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1
if err != nil {
t.Fatal(err)
}
if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
if f, err := forgemsg.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
t.Errorf("%s: %+v err %v", repo, f, err)
}
}
@@ -548,20 +555,20 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
t.Fatalf("FORGE holds %d messages (%v), want %d", info.State.Msgs, err, want)
}
}
named := func(seq uint64) []delivery.OpenPull {
named := func(seq uint64) []forgemsg.OpenPull {
t.Helper()
raw, err := forge.GetMsg(b.ctx, seq)
if err != nil || raw.Subject != openSubject {
t.Fatalf("message %d: %v %v, want the open message", seq, raw, err)
}
f, err := delivery.DecodeForge(raw.Data)
f, err := forgemsg.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
return f.Open
}
bothOpen := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
same := func(got []delivery.OpenPull) bool {
bothOpen := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
same := func(got []forgemsg.OpenPull) bool {
return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1]
}
@@ -713,7 +720,7 @@ func forgeMerged(t *testing.T, b *bed) []string {
if err != nil {
continue
}
if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged {
if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged {
out = append(out, f.Repository)
}
}
@@ -21,9 +21,11 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable
@@ -68,6 +70,12 @@ type publisher interface {
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
handleGrouped(ctx, m, pub, trees, files, kept, nil, logger)
}
// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names
// none), and the member's merged message written may end the group.
func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) {
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
switch {
case errors.Is(err, bridge.ErrRetry):
@@ -88,10 +96,18 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
if id := m.Headers().Get("x-event-id"); id != "" {
opts = append(opts, jetstream.WithMsgID(id))
}
var member forgemsg.Forge
grouped := false
if g != nil {
msg.Data, member, grouped = g.member(ctx, msg.Data)
}
if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil {
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
return
}
if grouped {
g.published(ctx, member)
}
if msg.Said != "" {
logger.Printf("%s: %s", m.Subject(), msg.Said)
}
@@ -136,17 +152,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
// A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges.
groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 3)
for _, w := range []struct {
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) {
perf.merges(ctx, key, rev, value)
groups.ended(ctx, key, rev, value)
}}} {
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
if kv == nil {
continue
}
if w.bucket == delivery.MergesBucket {
groups.merges = kvMerges(kv)
}
go func() {
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
@@ -155,6 +179,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
}
// A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and
// answered there; a state not there at this start is answered from the next start.
if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil {
go func() {
err := watch(ctx, kv, func(key string, rev uint64, value []byte) { groups.endAsked(ctx, kv, key, rev, value) })
if err != nil && ctx.Err() == nil {
logger.Printf("the end asks are no longer watched, so bridge_group_end_again is not answered until the bridge starts again: %v", err)
}
}()
}
if err := said(ctx); err != nil {
return err
}
@@ -171,7 +205,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Bounded: the turn is held, and no sync or other event goes on without it.
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
defer cancel()
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger)
})
}, logger.Printf)
if err != nil {
@@ -18,7 +18,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"mesh-delivery-forge-bridge/internal/bridge"
)
@@ -85,7 +85,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
t.Fatalf("published on %v", pub.subjects)
}
f, err := delivery.DecodeForge(pub.data[0])
f, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown {
t.Errorf("message %+v err %v", f, err)
}
@@ -107,11 +107,10 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") {
t.Errorf("log %q names neither the subject nor why", logs.String())
}
}
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
// 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here
// from a test of its own, for the test budget.
logs.Reset()
m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
if m.acked != 0 || m.termd != 0 {
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
@@ -173,7 +172,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" {
t.Fatalf("published on %v", pub.subjects)
}
f, err := delivery.DecodeForge(pub.data[0])
f, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" {
t.Errorf("message %+v err %v", f, err)
}
@@ -10,7 +10,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"mesh-delivery-forge-bridge/internal/bridge"
)
@@ -41,7 +41,7 @@ func (r *recorder) Publish(ctx context.Context, subject string, payload []byte,
if err != nil {
return ack, err
}
f, derr := delivery.DecodeForge(payload)
f, derr := forgemsg.DecodeForge(payload)
if derr != nil {
return ack, nil
}
@@ -101,7 +101,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore,
}
var differing []bridge.Message
for _, m := range msgs {
f, err := delivery.DecodeForge(m.Data)
f, err := forgemsg.DecodeForge(m.Data)
if err != nil {
differing = append(differing, m)
continue
@@ -13,7 +13,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
)
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
@@ -43,7 +43,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
}
want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"}
for i, w := range want {
f, err := delivery.DecodeForge(pub.data[i])
f, err := forgemsg.DecodeForge(pub.data[i])
if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err)
}
@@ -112,8 +112,8 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T)
if pub.subjects[1] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" {
t.Fatalf("last message on %s, want the open message after the pull messages", pub.subjects[1])
}
f, err := delivery.DecodeForge(pub.data[1])
want := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
f, err := forgemsg.DecodeForge(pub.data[1])
want := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || len(f.Open) != 2 || f.Open[0] != want[0] || f.Open[1] != want[1] {
t.Errorf("open message = %+v err %v, want it to name %v, the pull request whose head did not differ among them", f, err, want)
}
@@ -140,10 +140,10 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
pub := &fakePub{}
heads := newHeads()
r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)}
write := func(f delivery.Forge) {
write := func(f forgemsg.Forge) {
t.Helper()
ev, _ := delivery.ForgeEvent(f.Repository, f.Kind)
raw, err := delivery.EncodeForge(f)
ev, _ := forgemsg.ForgeEvent(f.Repository, f.Kind)
raw, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
@@ -151,19 +151,19 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
t.Fatal(err)
}
}
p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}
p := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated}
write(p)
if heads.get("novox.mesh-delivery.12") != head {
t.Fatalf("bucket %v", heads.vals)
}
p.Action = delivery.PullClosed
p.Action = forgemsg.PullClosed
write(p)
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
t.Errorf("a closed pull request is still kept: %v", heads.vals)
}
p.Action = delivery.PullUpdated
p.Action = forgemsg.PullUpdated
write(p)
write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
write(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
t.Errorf("a merged pull request is still kept: %v", heads.vals)
}
@@ -0,0 +1,413 @@
// Package forgemsg is the forge bridge's contract on its own stream `forge` (novox/hq ADR 0633 §4): its messages (Forge,
// one per pull request, merge, refusal or list of open pull requests) and a delivery group's end (ForgeGroupEnd). The
// forge bridge writes them; the indexer and the gate read them. It is the forge bridge's own package: one module owns
// these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
package forgemsg
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Version is the version of every value this package writes and the only one it reads: mesh-sdk's delivery values'.
const Version = delivery.Version
// ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names.
const ForgeStream = "forge"
// The kinds of forge message, the last part of a forge event's name.
const (
ForgePull = "pull"
ForgeMerged = "merged"
ForgeRefused = "refused"
ForgeOpen = "open"
)
// OpenPull names an open pull request into the trunk: the repository as `<owner>/<name>` and the number.
type OpenPull struct {
Repository string `json:"repository"`
Number int `json:"number"`
}
// OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of
// every repository the forge bridge reads (novox/hq ADR 0443 §5).
func OpenEvent() string { return "forge." + ForgeOpen }
// The actions of a pull message.
const (
PullOpened = "opened"
PullUpdated = "updated"
PullClosed = "closed"
)
// checkRepository refuses a repository that is not `<owner>/<name>`: exactly one slash, each part a valid name.
func checkRepository(repository string) error {
owner, name, ok := strings.Cut(repository, "/")
if !ok || strings.Contains(name, "/") {
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
}
if err := checkName("repository owner", owner); err != nil {
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
}
if err := checkName("repository name", name); err != nil {
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
}
if strings.Contains(owner, "/") {
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
}
return nil
}
func checkForgeKind(kind string) error {
switch kind {
case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen:
return nil
}
return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind)
}
// ForgeEvent is the name of a forge message of the given kind: `forge.<owner>.<name>.<kind>`.
func ForgeEvent(repository, kind string) (string, error) {
if err := checkForgeKind(kind); err != nil {
return "", err
}
if kind == ForgeOpen {
return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent())
}
if err := checkRepository(repository); err != nil {
return "", err
}
owner, name, _ := strings.Cut(repository, "/")
return "forge." + owner + "." + name + "." + kind, nil
}
// ParseForgeEvent is the inverse of ForgeEvent.
func ParseForgeEvent(event string) (repository, kind string, err error) {
if event == OpenEvent() {
return "", ForgeOpen, nil
}
parts := strings.Split(event, ".")
if len(parts) != 4 || parts[0] != "forge" {
return "", "", fmt.Errorf("delivery: %q is not a forge event, forge.<owner>.<name>.<kind>", event)
}
repository = parts[1] + "/" + parts[2]
if _, err := ForgeEvent(repository, parts[3]); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err)
}
return repository, parts[3], nil
}
// Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull
// message says Action; a merged message carries the merge commit, the tree of every directory holding a
// module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own
// words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and
// DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither
// (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither.
type Forge struct {
V int `json:"v"`
Kind string `json:"kind"`
Repository string `json:"repository"`
Branch string `json:"branch,omitempty"`
Number int `json:"number,omitempty"`
Head string `json:"head,omitempty"`
Action string `json:"action,omitempty"`
Dirs []string `json:"dirs,omitempty"`
DirsKnown bool `json:"dirs-known,omitempty"`
Merge string `json:"merge,omitempty"`
Trees map[string]string `json:"trees,omitempty"`
Root string `json:"root,omitempty"`
// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
Manifests map[string]string `json:"manifests,omitempty"`
Locks map[string]map[string]string `json:"locks,omitempty"`
// Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
// indexer names the paths; a message without them gives such a module none. Older writers send none.
Paths map[string]map[string]string `json:"paths,omitempty"`
// Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573
// §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its
// fingerprint. Absent for a directory that uses none, and from an older writer.
Packages map[string][]string `json:"packages,omitempty"`
Act string `json:"act,omitempty"`
Words string `json:"words,omitempty"`
Open []OpenPull `json:"open,omitempty"`
// RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is
// the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is
// refused, loudly, rather than read with them dropped.
RetiredPlacements json.RawMessage `json:"placements,omitempty"`
RetiredRead bool `json:"placements-read,omitempty"`
// Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565
// §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds
// no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it
// held.
Predefined json.RawMessage `json:"predefined,omitempty"`
PredefinedRead bool `json:"predefined-read,omitempty"`
// Group is the delivery group a merged message's pull request was merged in, and Members every approved member of
// that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the
// gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge
// with no group carries neither, and an older bridge sends neither; an older reader ignores both.
Group string `json:"group,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
}
// checkGroup refuses a group on anything but a merged message, a group without its members or members without a
// group, a member that is not a repository's pull request at a head, a member named twice, and members that leave out
// the message's own pull request.
func (f Forge) checkGroup() error {
if f.Group == "" && len(f.Members) == 0 {
return nil
}
if f.Kind != ForgeMerged {
return fmt.Errorf("delivery: only a merged message names its delivery group, not a %s message", f.Kind)
}
if f.Group == "" || len(f.Members) == 0 {
return fmt.Errorf("delivery: a merged message names its group and the group's members together, or neither")
}
self := false
seen := map[string]bool{}
for _, m := range f.Members {
if err := checkRepository(m.Repository); err != nil {
return err
}
if m.Number <= 0 || m.Head == "" {
return fmt.Errorf("delivery: the member %s of the group %q needs its pull request number and head", m.Repository, f.Group)
}
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
if seen[k] {
return fmt.Errorf("delivery: the group %q names its member %s twice", f.Group, k)
}
seen[k] = true
self = self || (delivery.SameRepository(m.Repository, f.Repository) && m.Number == f.Number)
}
if !self {
return fmt.Errorf("delivery: the members of the group %q leave out the merged pull request %s#%d", f.Group, f.Repository, f.Number)
}
return nil
}
// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and
// a list said both present and absent.
func (f Forge) checkPredefined() error {
if len(f.Predefined) == 0 && !f.PredefinedRead {
return nil
}
if f.Kind != ForgeMerged {
return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind)
}
if len(f.Predefined) > 0 && f.PredefinedRead {
return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)")
}
var list map[string]json.RawMessage
if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil {
return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object")
}
return nil
}
func (f Forge) check() error {
if err := checkForgeKind(f.Kind); err != nil {
return err
}
if f.Kind == ForgeOpen {
return f.checkOpen()
}
if len(f.Open) > 0 {
return fmt.Errorf("delivery: only an open message names open pull requests")
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead {
return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " +
"placement is the seat mesh-placement's state")
}
if err := f.checkPredefined(); err != nil {
return err
}
if err := f.checkGroup(); err != nil {
return err
}
if err := checkRepository(f.Repository); err != nil {
return err
}
switch f.Kind {
case ForgePull:
if f.Number <= 0 {
return fmt.Errorf("delivery: a pull message needs its pull request number")
}
if f.Head == "" {
return fmt.Errorf("delivery: a pull message needs its head")
}
switch f.Action {
case PullOpened, PullUpdated, PullClosed:
default:
return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action)
}
if len(f.Dirs) > 0 && !f.DirsKnown {
return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
}
dirs := map[string]bool{}
for _, d := range f.Dirs {
dirs[d] = true
}
if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
return err
}
case ForgeMerged:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a merged message carries no module directories")
}
if f.Number <= 0 {
return fmt.Errorf("delivery: a merged message needs its pull request number")
}
if f.Head == "" {
return fmt.Errorf("delivery: a merged message needs its head")
}
if f.Merge == "" {
return fmt.Errorf("delivery: a merged message needs its merge commit")
}
if f.Root == "" {
return fmt.Errorf("delivery: a merged message needs the repository's root tree")
}
// No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one
// whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the
// repository's modules (novox/hq issue 1126). An older writer always sent at least one tree, so none is unambiguous;
// such a message carries no manifests, locks or paths (checkInputs below: each names a directory of its trees).
for dir, tree := range f.Trees {
if tree == "" {
return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
}
}
if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
return err
}
case ForgeRefused:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a refused message carries no module directories")
}
if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 {
return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths")
}
if f.Act == "" {
return fmt.Errorf("delivery: a refused message needs the act it answers")
}
if f.Words == "" {
return fmt.Errorf("delivery: a refused message needs the forge's words")
}
}
return nil
}
// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
// by), an empty manifest and an empty lock path or hash, naming the directory.
func (f Forge) checkInputs(known func(dir string) bool, of string) error {
for dir, m := range f.Manifests {
if !known(dir) {
return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
}
if m == "" {
return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
}
}
for dir, locks := range f.Locks {
if !known(dir) {
return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
}
for path, hash := range locks {
if path == "" || hash == "" {
return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
}
}
}
for dir, paths := range f.Paths {
if !known(dir) {
return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of)
}
for path, object := range paths {
if path == "" || object == "" {
return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir)
}
}
}
return nil
}
// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
// an open pull request that is not `<owner>/<name>` and a number.
func (f Forge) checkOpen() error {
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" || f.Group != "" || len(f.Members) > 0 {
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
}
for _, p := range f.Open {
if err := checkRepository(p.Repository); err != nil {
return err
}
if p.Number <= 0 {
return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository)
}
}
return nil
}
// EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs.
func EncodeForge(f Forge) ([]byte, error) {
if err := f.check(); err != nil {
return nil, err
}
f.V = Version
return json.Marshal(f)
}
// DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses.
func DecodeForge(data []byte) (Forge, error) {
var f Forge
if err := decode(data, &f); err != nil {
return Forge{}, err
}
if err := f.check(); err != nil {
return Forge{}, err
}
return f, nil
}
// SameForge says whether two forge messages are the same value.
func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }
var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`)
// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses
// (mesh-sdk delivery's rule for a name).
func checkName(kind, name string) error {
if name == "" {
return fmt.Errorf("delivery: the %s name is empty", kind)
}
if !nameOK.MatchString(name) {
return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name)
}
return nil
}
// decode reads a value of Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk delivery's
// values are read.
func decode(data []byte, into any) error {
var probe struct {
V *int `json:"v"`
}
if err := json.Unmarshal(data, &probe); err != nil {
return fmt.Errorf("delivery: value is not JSON: %w", err)
}
if probe.V == nil {
return fmt.Errorf("%w: no version in the value, this reader reads %d", delivery.ErrUnknownVersion, Version)
}
if *probe.V != Version {
return fmt.Errorf("%w: the value is version %d, this reader reads %d", delivery.ErrUnknownVersion, *probe.V, Version)
}
if err := json.Unmarshal(data, into); err != nil {
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
}
return nil
}
@@ -0,0 +1,193 @@
package forgemsg
import (
"encoding/json"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — <b> & \\n\"\n}\n\n"
func sampleForges() []Forge {
return []Forge{
{Kind: ForgePull, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 36, Head: "aaaa", Action: "opened"},
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc",
Trees: map[string]string{"modules/a": "t1"}, Root: "t0"},
{Kind: ForgeRefused, Repository: "novox/mesh-sdk", Act: "novox/mesh-sdk.36.aaaa.view", Words: "403 forbidden"},
{Kind: ForgeOpen, Open: []OpenPull{{Repository: "novox/mesh-sdk", Number: 36}, {Repository: "novox/mesh-control", Number: 7}}},
// 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4).
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0",
Group: "feat/take-in", Members: []delivery.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}},
}
}
func mergedWithInputs() Forge {
return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0",
Trees: map[string]string{"a": "t1", "b": "t2"},
Manifests: map[string]string{"a": oddManifest},
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}}
}
// TestForge holds the forge bridge's contract, moved whole from mesh-sdk go/delivery with its tests (1200/AC1, AC3,
// AC4; 1279/AC5; 1442/AC1-AC4; 1585/T1).
func TestForge(t *testing.T) {
t.Run("events", func(t *testing.T) {
ev, err := ForgeEvent("novox/mesh-sdk", ForgeMerged)
if err != nil || ev != "forge.novox.mesh-sdk.merged" {
t.Fatalf("forge event = %q, %v", ev, err)
}
if repo, kind, err := ParseForgeEvent(ev); err != nil || repo != "novox/mesh-sdk" || kind != ForgeMerged {
t.Fatalf("parse forge event = %q %q %v", repo, kind, err)
}
for _, bad := range []struct{ repo, kind string }{
{"", "pull"}, {"novox", "pull"}, {"a/b/c", "pull"}, {"/b", "pull"}, {"a/", "pull"},
{"a.b/c", "pull"}, {"a/b c", "pull"}, {"a/b", "closed"}, {"a/b", ""}, {"a/b", ForgeOpen},
} {
if _, err := ForgeEvent(bad.repo, bad.kind); err == nil {
t.Errorf("forge event %q %q accepted", bad.repo, bad.kind)
}
}
for _, bad := range []string{"", "forge.a.b", "forge.a.b.closed", "x.a.b.pull", "forge.a.b.pull.x", "forge..b.pull", ForgeGroupEndEvent} {
if _, _, err := ParseForgeEvent(bad); err == nil {
t.Errorf("parse forge event %q accepted", bad)
}
}
if repo, kind, err := ParseForgeEvent(OpenEvent()); OpenEvent() != "forge.open" || err != nil || repo != "" || kind != ForgeOpen {
t.Errorf("open event %q = %q %q %v", OpenEvent(), repo, kind, err)
}
})
t.Run("round trip", func(t *testing.T) {
fs := append(sampleForges(), mergedWithInputs())
for _, f := range fs {
data, err := EncodeForge(f)
if err != nil {
t.Fatalf("%s: %v", f.Kind, err)
}
var probe map[string]any
if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) {
t.Fatalf("%s: json %s does not carry v:1", f.Kind, data)
}
f.V = Version
if got, err := DecodeForge(data); err != nil || !SameForge(got, f) {
t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err)
}
}
// An empty list is an open message too: nothing is open.
if data, err := EncodeForge(Forge{Kind: ForgeOpen, Open: []OpenPull{}}); err != nil {
t.Errorf("empty open: %v", err)
} else if f, err := DecodeForge(data); err != nil || f.Kind != ForgeOpen || len(f.Open) != 0 {
t.Errorf("empty open round trip: %+v %v", f, err)
}
ge := ForgeGroupEnd{Group: "feat/take-in", Members: []GroupMerge{{Repository: "novox/mesh-sdk", Number: 37, Merge: "eeee"}}}
data, err := EncodeForgeGroupEnd(ge)
ge.V = Version
if got, err2 := DecodeForgeGroupEnd(data); err != nil || err2 != nil || !SameForgeGroupEnd(got, ge) {
t.Errorf("group end round trip: %+v %v %v", got, err, err2)
}
// An older bridge's message reads, and is written back byte for byte.
old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}`
if f, err := DecodeForge([]byte(old)); err != nil || f.Manifests != nil {
t.Fatalf("%+v, %v", f, err)
} else if data, err := EncodeForge(f); err != nil || string(data) != old {
t.Errorf("%s, %v, want exactly %s", data, err, old)
}
// A merge leaving the repository with no module.json is a message too (issue 1126).
f := sampleForges()[1]
f.Trees = nil
if data, err := EncodeForge(f); err != nil {
t.Errorf("a merged message with no trees is refused: %v", err)
} else if got, err := DecodeForge(data); err != nil || len(got.Trees) != 0 {
t.Errorf("a merged message with no trees comes back as %+v, %v", got, err)
}
for name, dec := range map[string]func([]byte) error{
"forge": func(b []byte) error { _, err := DecodeForge(b); return err },
"group end": func(b []byte) error { _, err := DecodeForgeGroupEnd(b); return err },
} {
for _, in := range []string{`{"group":"g"}`, `{"v":2,"group":"g"}`} {
if err := dec([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) {
t.Errorf("%s decode of %s = %v, want delivery.ErrUnknownVersion", name, in, err)
}
}
}
})
t.Run("refused", func(t *testing.T) {
at := func(i int, mut func(*Forge)) Forge { f := sampleForges()[i]; mut(&f); return f }
in := func(mut func(*Forge)) Forge { f := mergedWithInputs(); mut(&f); return f }
pullIn := func(mut func(*Forge)) Forge {
f := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened,
Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest},
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}}
mut(&f)
return f
}
for _, c := range []struct {
name, want string
f Forge
}{
{"merged without merge commit", "merge", at(1, func(f *Forge) { f.Merge = "" })},
{"merged without root", "root", at(1, func(f *Forge) { f.Root = "" })},
{"merged with a manifest and no trees", "modules/a", at(1, func(f *Forge) { f.Trees = nil; f.Manifests = map[string]string{"modules/a": "{}"} })},
{"merged with an empty tree", "modules/a", at(1, func(f *Forge) { f.Trees = map[string]string{"modules/a": ""} })},
{"pull with an unknown action", "reopened", at(0, func(f *Forge) { f.Action = "reopened" })},
{"pull without head", "head", at(0, func(f *Forge) { f.Head = "" })},
{"refused without words", "words", at(2, func(f *Forge) { f.Words = "" })},
{"refused without act", "act", at(2, func(f *Forge) { f.Act = "" })},
{"unknown kind", "closed", Forge{Kind: "closed", Repository: "a/b"}},
{"bad repository", "nope", Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}},
{"open with a repository", "only the open", Forge{Kind: ForgeOpen, Repository: "a/b", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
{"open with a bad entry", "", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a", Number: 1}}}},
{"open with a zero number", "no number", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a/b"}}}},
{"pull naming open pull requests", "only an open message", Forge{Kind: ForgePull, Repository: "a/b", Number: 1, Head: "h", Action: "updated", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
{"manifest outside trees", `"zz"`, in(func(f *Forge) { f.Manifests["zz"] = "{}" })},
{"locks outside trees", `"yy"`, in(func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} })},
{"empty manifest", `"b"`, in(func(f *Forge) { f.Manifests["b"] = "" })},
{"empty lock hash", `"a"`, in(func(f *Forge) { f.Locks["a"]["go.sum"] = "" })},
{"named paths outside trees", `"other"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"other": {"src": "o"}} })},
{"a named path with no object", `"src"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"a": {"src": ""}} })},
{"placement files", "mesh-placement", in(func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) })},
{"placements read", "mesh-placement", in(func(f *Forge) { f.RetiredRead = true })},
{"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })},
{"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })},
{"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}},
{"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })},
{"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })},
{"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}},
{"open with locks", "only the open", Forge{Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}},
// 1585/T1: a group only on a merged message, with its members, the message's own pull request among them.
{"pull with a group", "only a merged message names its delivery group", at(0, func(f *Forge) {
f.Group, f.Members = "g", []delivery.Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}}
})},
{"merged with a group and no members", "together", at(4, func(f *Forge) { f.Members = nil })},
{"merged with members and no group", "together", at(4, func(f *Forge) { f.Group = "" })},
{"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })},
{"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })},
{"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}},
} {
if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
}
}
// A member's repository in another case still names the merged pull request (10c91fba).
f := sampleForges()[4]
f.Members[0].Repository = "Novox/Mesh-SDK"
if _, err := EncodeForge(f); err != nil {
t.Errorf("a member in another case: %v", err)
}
for _, c := range []struct {
name, want string
e ForgeGroupEnd
}{
{"group end without members", "no member", ForgeGroupEnd{Group: "g"}},
{"group end without a group", "names its group", ForgeGroupEnd{Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}}}},
{"group end with a member without its merge", "merge commit", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1}}}},
{"group end with a member twice", "twice", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}, {Repository: "a/b", Number: 1, Merge: "m"}}}},
} {
if _, err := EncodeForgeGroupEnd(c.e); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
}
}
})
}
@@ -0,0 +1,5 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
@@ -0,0 +1,14 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
@@ -0,0 +1,78 @@
package forgemsg
import (
"encoding/json"
"fmt"
"reflect"
)
// ForgeGroupEndEvent is the name of a delivery group's end on the forge bridge's stream (novox/hq ADR 0633 §4): the
// bridge publishes it once every approved member of the group is merged, after the last member's merged message. It
// has three parts, so no repository's forge event (`forge.<owner>.<name>.<kind>`) is ever this name.
const ForgeGroupEndEvent = "forge.group.merged"
// GroupMerge is one merged member of a delivery group: the repository as `<owner>/<name>`, its pull request and the
// merge commit the forge made of it.
type GroupMerge struct {
Repository string `json:"repository"`
Number int `json:"number"`
Merge string `json:"merge"`
}
// ForgeGroupEnd is the body of ForgeGroupEndEvent: the group and every member merged in it, with its merge commit.
// The indexer takes the members it holds in as one source revision once it hears it (ADR 0633 §4).
type ForgeGroupEnd struct {
V int `json:"v"`
Group string `json:"group"`
Members []GroupMerge `json:"members"`
}
func (e ForgeGroupEnd) check() error {
if e.Group == "" {
return fmt.Errorf("delivery: a group's end names its group")
}
if len(e.Members) == 0 {
return fmt.Errorf("delivery: the end of the group %q names no member merged in it", e.Group)
}
seen := map[string]bool{}
for _, m := range e.Members {
if err := checkRepository(m.Repository); err != nil {
return err
}
if m.Number <= 0 || m.Merge == "" {
return fmt.Errorf("delivery: the member %s of the group %q's end needs its pull request number and merge commit", m.Repository, e.Group)
}
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
if seen[k] {
return fmt.Errorf("delivery: the end of the group %q names its member %s twice", e.Group, k)
}
seen[k] = true
}
return nil
}
// EncodeForgeGroupEnd is e as JSON at Version; it refuses an end with no group or no member, and a member without its
// pull request or merge commit.
func EncodeForgeGroupEnd(e ForgeGroupEnd) ([]byte, error) {
if err := e.check(); err != nil {
return nil, err
}
e.V = Version
return json.Marshal(e)
}
// DecodeForgeGroupEnd reads a group's end; it refuses an unknown version (ErrUnknownVersion) and what
// EncodeForgeGroupEnd refuses.
func DecodeForgeGroupEnd(data []byte) (ForgeGroupEnd, error) {
var e ForgeGroupEnd
if err := decode(data, &e); err != nil {
return ForgeGroupEnd{}, err
}
if err := e.check(); err != nil {
return ForgeGroupEnd{}, err
}
return e, nil
}
// SameForgeGroupEnd says whether two group ends are the same value.
func SameForgeGroupEnd(a, b ForgeGroupEnd) bool { return reflect.DeepEqual(a, b) }
+4 -1
View File
@@ -3,12 +3,13 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
@@ -20,3 +21,5 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ./forgemsg
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -5,6 +5,7 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -145,11 +146,11 @@ func MergeActs(value []byte) ([]Act, error) {
// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull
// request and head it was on, and the forge's own words.
func RefusedMessage(a Act, words string) (Message, error) {
event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused)
event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused)
if err != nil {
return Message{}, err
}
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number,
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number,
Head: a.Head, Act: a.Name, Words: words})
if err != nil {
return Message{}, err
@@ -5,6 +5,7 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -27,7 +28,7 @@ func owedValue(t *testing.T, repository string, number int, act string, o delive
return key, b
}
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1
key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"})
a, err := OwedAct(key, v)
@@ -42,6 +43,19 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" {
t.Errorf("act = %+v", a)
}
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
// performs no mesh/check act.
ck, cv := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
}
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
}
}
// checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed).
@@ -58,23 +72,6 @@ func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) {
return key, v
}
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
// performs no mesh/check act.
func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1
key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
a, err := OwedAct(key, v)
if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
}
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
}
}
func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2
cases := []struct {
act string
@@ -111,18 +108,12 @@ func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) {
}
}
func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) {
key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
a, err := OwedAct(key, v)
if err != nil {
t.Fatal(err)
}
if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
t.Errorf("act = %+v", a)
}
}
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
// A dotted repository's key is unescaped, and its act names the repository whole.
dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
t.Errorf("a dotted repository's act = %+v, %v", a, err)
}
key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"})
if _, err := OwedAct(key, v); err == nil {
@@ -166,13 +157,11 @@ func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { /
if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA {
t.Errorf("act = %+v", acts[0])
}
}
func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) {
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
// A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which.
v = mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}})
if _, err := MergeActs(v); err == nil {
t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which")
t.Error("a merge with no approved heads was taken")
}
}
@@ -185,13 +174,13 @@ func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3,
if m.Event != "forge.novox.mesh-delivery.refused" {
t.Errorf("event = %q", m.Event)
}
f, err := delivery.DecodeForge(m.Data)
f, err := forgemsg.DecodeForge(m.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"}
if !delivery.SameForge(f, want) {
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v", f)
}
}
@@ -15,7 +15,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
)
// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
@@ -110,11 +110,11 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
var action string
switch subject {
case UpdatedSubject:
action = delivery.PullUpdated
action = forgemsg.PullUpdated
case ClosedSubject:
action = delivery.PullClosed
action = forgemsg.PullClosed
case MergedSubject:
action = delivery.PullClosed // unless it merged into the trunk, below
action = forgemsg.PullClosed // unless it merged into the trunk, below
default:
return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
}
@@ -136,29 +136,29 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
return merged(subject, p, trees, files, kept)
}
said := ""
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk {
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
action = delivery.PullClosed
action = forgemsg.PullClosed
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
}
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: action}
// The directories are the forge's word only when it said it could read them (module_dirs_said).
if action == delivery.PullUpdated && p.ModuleDirsSaid {
if action == forgemsg.PullUpdated && p.ModuleDirsSaid {
f.Dirs, f.DirsKnown = p.ModuleDirs, true
var err error
if f.Manifests, f.Locks, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
body, err := delivery.EncodeForge(f)
body, err := forgemsg.EncodeForge(f)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
@@ -313,11 +313,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
if len(locks) == 0 {
locks = nil
}
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
if repository == CatalogRepository {
// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
@@ -331,7 +331,7 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
f.Predefined = json.RawMessage(raw)
}
}
body, err := delivery.EncodeForge(f)
body, err := forgemsg.EncodeForge(f)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
@@ -373,12 +373,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
continue
}
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: delivery.PullUpdated}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: forgemsg.PullUpdated}
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
var body []byte
if err == nil {
body, err = delivery.EncodeForge(f)
body, err = forgemsg.EncodeForge(f)
}
if err != nil {
skipped = append(skipped, name+": "+err.Error())
@@ -392,12 +392,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
func Track(f delivery.Forge) (key, head string, ok bool) {
func Track(f forgemsg.Forge) (key, head string, ok bool) {
key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
switch {
case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed:
return key, f.Head, true
case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged:
return key, "", true
}
return "", "", false
@@ -413,22 +413,22 @@ func OpenMessage(answer []byte) (Message, error) {
if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
}
open := []delivery.OpenPull{}
open := []forgemsg.OpenPull{}
for _, p := range *a.Pulls {
if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 {
continue
}
open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
}
slices.SortFunc(open, func(a, b delivery.OpenPull) int {
slices.SortFunc(open, func(a, b forgemsg.OpenPull) int {
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
return c
}
return a.Number - b.Number
})
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open})
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open})
if err != nil {
return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err)
}
return Message{Event: delivery.OpenEvent(), Data: body}, nil
return Message{Event: forgemsg.OpenEvent(), Data: body}, nil
}
@@ -6,6 +6,7 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -54,13 +55,13 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
if msg.Event != "forge.novox.mesh-delivery.pull" {
t.Errorf("event = %q", msg.Event)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !delivery.SameForge(f, want) {
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
}
@@ -70,7 +71,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
f, _ := delivery.DecodeForge(msg.Data)
f, _ := forgemsg.DecodeForge(msg.Data)
if f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
}
@@ -79,7 +80,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
f, _ = delivery.DecodeForge(msg.Data)
f, _ = forgemsg.DecodeForge(msg.Data)
if !f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
}
@@ -88,7 +89,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
if f, _ = forgemsg.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown)
}
}
@@ -101,13 +102,13 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
if msg.Event != "forge.novox.mesh-delivery.pull" {
t.Errorf("event = %q", msg.Event)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullClosed}
if !delivery.SameForge(f, want) {
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullClosed}
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
@@ -116,12 +117,12 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
}
for _, base := range []any{"main", nil} {
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" {
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
}
}
@@ -170,12 +171,18 @@ func mustDrop(body []byte, key string) []byte {
return b
}
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
}
}
// 1236: a merged event without a full merge commit is unreadable, and the forge is not asked.
var calls []call
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls)
}
}
const merge = "89abcdef0123456789abcdef0123456789abcdef"
@@ -222,13 +229,13 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
if msg.Event != "forge.novox.mesh-delivery.merged" {
t.Errorf("event = %q", msg.Event)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
if !delivery.SameForge(f, want) {
if !forgemsg.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) {
@@ -242,13 +249,13 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullClosed}
if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) {
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullClosed}
if msg.Event != "forge.novox.mesh-delivery.pull" || !forgemsg.SameForge(f, want) {
t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f)
}
if len(calls) != 0 {
@@ -277,22 +284,14 @@ func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) {
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
f, err := forgemsg.DecodeForge(msg.Data)
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Merge: merge, Root: "r0"}
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err)
}
}
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
var calls []call
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
t.Errorf("err = %v calls = %v", err, calls)
}
}
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry
// (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}.
func openAnswer(pulls ...map[string]any) []byte {
@@ -311,12 +310,12 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
if err != nil || len(skipped) != 0 || len(msgs) != 2 {
t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err)
}
for i, want := range []delivery.Forge{
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated},
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated},
for i, want := range []forgemsg.Forge{
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated},
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated},
} {
f, err := delivery.DecodeForge(msgs[i].Data)
if err != nil || !delivery.SameForge(f, want) {
f, err := forgemsg.DecodeForge(msgs[i].Data)
if err != nil || !forgemsg.SameForge(f, want) {
t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want)
}
if f.DirsKnown {
@@ -328,14 +327,11 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
}
}
func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)))
if err != nil || len(msgs) != 0 || len(skipped) != 0 {
t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4
// 1237/AC4: an open pull request into another branch gets no message.
if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 {
t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
}
}
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237
msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)))
if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") {
t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err)
@@ -355,15 +351,15 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
if err != nil || m.Event != "forge.open" {
t.Fatalf("message %+v err %v", m, err)
}
f, err := delivery.DecodeForge(m.Data)
f, err := forgemsg.DecodeForge(m.Data)
// Every one into the trunk, whether its head reads or not (it is open); none into another branch; in a fixed order.
want := []delivery.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
want := []forgemsg.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
t.Errorf("open message = %+v err %v, want %v", f, err, want)
}
// Nothing open is a message too: the indexer then drops every candidate.
m, err = OpenMessage([]byte(`{"pulls":[]}`))
if f, derr := delivery.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 0 {
if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 {
t.Errorf("empty answer: %+v %v %v", f, err, derr)
}
for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} {
@@ -374,24 +370,24 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
}
func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6
pullMsg := func(action string) delivery.Forge {
return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
pullMsg := func(action string) forgemsg.Forge {
return forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
}
for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} {
for _, a := range []string{forgemsg.PullOpened, forgemsg.PullUpdated} {
if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok {
t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok)
}
}
// A closed pull request and a merged one are no longer kept: their key goes.
if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
if key, h, ok := Track(pullMsg(forgemsg.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
t.Errorf("closed: Track = %q %q %v", key, h, ok)
}
merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
merged := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok {
t.Errorf("merged: Track = %q %q %v", key, h, ok)
}
// The open message and the rest hold no head.
if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok {
if _, _, ok := Track(forgemsg.Forge{Kind: forgemsg.ForgeOpen}); ok {
t.Error("an open message was tracked")
}
}
@@ -0,0 +1,79 @@
package bridge
import (
"context"
"encoding/json"
"fmt"
"regexp"
)
// Caller calls a module's tool by its manifest key (`<module>.<tool>`) with a JSON body and returns the tool's result:
// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask).
type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error)
// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
const CommitTreesTool = "gitea.gitea_commit_trees"
// TreesVia asks the forge for a commit's trees through call.
func TreesVia(ctx context.Context, call Caller) Trees {
return func(owner, repo, sha string) (string, map[string]string, error) {
raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return "", nil, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err)
}
if a.Root == "" {
return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool)
}
return a.Root, a.Trees, nil
}
}
// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
const GetFileTool = "gitea.gitea_get_file"
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
// recognises it as /: 404 /).
var notFound = regexp.MustCompile(`: 404 `)
// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound.
func FilesVia(ctx context.Context, call Caller) Files {
return func(owner, repo, path, ref string) ([]byte, error) {
raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
if notFound.MatchString(err.Error()) {
return nil, fmt.Errorf("%w: %v", ErrNotFound, err)
}
return nil, err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err)
}
return []byte(*a.File.Content), nil
}
}
// MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes
// on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head
// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs).
func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) {
body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge})
if err != nil {
return Message{}, err
}
return ConvertKept(MergedSubject, body, trees, files, nil)
}
@@ -13,7 +13,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
)
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
@@ -69,7 +69,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
@@ -90,7 +90,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil {
if f, err = forgemsg.DecodeForge(msg.Data); err != nil {
t.Fatal(err)
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
@@ -108,7 +108,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
}
forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`)
msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
t.Errorf("predefined = %s (%v)", f.Predefined, err)
}
forge.calls = nil
@@ -133,7 +133,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if err != nil {
t.Fatal(err)
}
f, _ := delivery.DecodeForge(msg.Data)
f, _ := forgemsg.DecodeForge(msg.Data)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 {
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
}
@@ -146,7 +146,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if err != nil {
t.Fatal(err)
}
f, _ = delivery.DecodeForge(msg.Data)
f, _ = forgemsg.DecodeForge(msg.Data)
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
@@ -175,7 +175,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin
if err != nil {
t.Fatal(err)
}
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
if f, _ := forgemsg.DecodeForge(msg.Data); f.Locks != nil {
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
}
}
@@ -4,7 +4,7 @@ import (
"os"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
)
// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a
@@ -26,7 +26,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) {
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
@@ -0,0 +1,198 @@
// Package groups is what the forge bridge's process and its tools bundle share of a delivery group's end (novox/hq ADR
// 0633 §4, issue 1585): the keys of the bridge's state performed, and the asks of the state end-asks, through which the
// tool bridge_group_end_again asks the process to say a group's end again (the bridge serves no request subject, ADR 0437
// §4, so a tool asks in a state the process watches, as indexer_take_in asks the indexer in takeins).
package groups
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The bridge's states the tools read and write.
const (
// PerformedState is the bridge's state of what it performed and published.
PerformedState = "performed"
// EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group.
EndAsksState = "end-asks"
// EndAsksBucket is its bucket on the bus.
EndAsksBucket = delivery.ForgeBridgeModule + "_" + EndAsksState
)
// Key is the key of performed that says a group's end is written: `groups.<merges key>`, its value the revision of the
// gate's merges key it was worked out from.
func Key(mergesKey string) string { return "groups." + mergesKey }
// PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the
// sha256 of <repository>#<number>@<merge>>`.
func PublishedKey(repository string, number int, merge string) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge)))
return "published." + hex.EncodeToString(sum[:8])
}
// EndID is the message id of a group's end, and of one said again by hand at a time: the stream's duplicate window drops a
// second publish of the same id.
func EndID(group, lastMerge, again string) string {
id := "group-end." + group + "." + lastMerge
if again != "" {
id += "." + again
}
return id
}
// AgainStamp is the time a hand ask appends to a group end's message id.
func AgainStamp(at time.Time) string { return at.UTC().Format("20060102T150405.000000000Z") }
// EndAsk is a value of the end-asks state: which group's end a person asked to be said again, why, who and when, and, once
// the bridge read it, its answer.
type EndAsk struct {
V int `json:"v"`
Group string `json:"group"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Written bool `json:"written,omitempty"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// Done says the ask is answered.
func (a EndAsk) Done() bool { return !a.Answered.IsZero() }
// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value.
func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) {
group = strings.TrimSpace(group)
if group == "" {
return "", nil, errors.New("name the group whose end is said again (group)")
}
key, err := delivery.MergesKey(group)
if err != nil {
return "", nil, err
}
data, err := json.Marshal(EndAsk{V: 1, Group: group, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// DecodeEndAsk reads an end-asks value.
func DecodeEndAsk(data []byte) (EndAsk, error) {
var a EndAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 || a.Group == "" {
return a, errors.New("an end-asks value of no version 1 naming no group")
}
return a, nil
}
// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end.
type Row struct {
Key string `json:"key"`
Group string `json:"group"`
Members []RowMember `json:"members"`
Ended bool `json:"ended"`
EndedAt uint64 `json:"ended-at-revision,omitempty"`
Open bool `json:"open"`
Words string `json:"words"`
// LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the
// bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group.
LastEndAsk *EndAsk `json:"last-end-ask,omitempty"`
}
// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written.
type RowMember struct {
Repository string `json:"repository"`
Number int `json:"number"`
State string `json:"state"`
Commit string `json:"commit,omitempty"`
Published bool `json:"published"`
}
// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end
// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its
// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's
// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as
// `end-asks.<key>`.
func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) {
lastAsk := map[string]*EndAsk{}
for key, raw := range asks {
a, err := DecodeEndAsk(raw)
if err != nil {
unread = append(unread, EndAsksState+"."+key)
continue
}
lastAsk[key] = &a
}
for key, raw := range merges {
m, err := delivery.DecodeMerge(raw)
if err != nil {
unread = append(unread, key)
continue
}
if len(m.Approved) < 2 {
continue
}
r := Row{Key: key, Group: m.Group}
allMerged := true
for _, p := range m.Pulls {
pub := p.Commit != "" && performed[PublishedKey(p.Repository, p.Number, p.Commit)]
r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: p.Number, State: string(p.State), Commit: p.Commit, Published: pub})
if p.State != delivery.MergeMerged || !pub {
allMerged = false
}
}
if len(m.Pulls) < len(m.Approved) {
allMerged = false
}
r.EndedAt, r.Ended = endedAt[key]
r.LastEndAsk = lastAsk[key]
r.Open = !r.Ended
switch {
case r.Ended:
r.Words = "its end is written"
case allMerged:
r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it"
default:
r.Words = "gathered: not every member is merged and its merged message written"
}
rows = append(rows, r)
}
sortRows(rows)
sort.Strings(unread)
return rows, unread
}
func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) }
// MembersOf are a merge's approved members with their heads, in the order of their keys.
func MembersOf(m delivery.Merge) []delivery.Head {
var out []delivery.Head
for _, k := range slices.Sorted(func(yield func(string) bool) {
for k := range m.Approved {
if !yield(k) {
return
}
}
}) {
at := strings.LastIndex(k, "@")
if at < 0 {
continue
}
n, err := strconv.Atoi(k[at+1:])
if err != nil {
continue
}
out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]})
}
return out
}
@@ -28,12 +28,13 @@ type state struct {
}
type artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
}
type manifest struct {
@@ -44,6 +45,7 @@ type manifest struct {
Streams []stream `json:"streams"`
State []state `json:"state"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
Invokes []string `json:"invokes"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
@@ -107,11 +109,15 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
// the forge only for the directories it changed (issue 1126).
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585
t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State)
}
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
// 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585).
if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status).
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the bridge has none", k)
}
@@ -143,8 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
+18 -1
View File
@@ -13,7 +13,13 @@
"state": [
{ "name": "heads", "history": 1 },
{ "name": "performed", "history": 1 },
{ "name": "inputs", "history": 1 }
{ "name": "inputs", "history": 1 },
{ "name": "end-asks", "history": 1 }
],
"tools": [
"bridge_groups",
"bridge_group_end_again",
"bridge_merged_message"
],
"reads": [
"mesh-delivery-gate.merges",
@@ -75,6 +81,17 @@
"system": "arch",
"from": "cmd/mesh-delivery-forge-bridge",
"binary": "mesh-delivery-forge-bridge"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-forge-bridge-tools",
"binary": "mesh-delivery-forge-bridge-tools",
"loads": [
"mesh-delivery-forge-bridge-tools"
]
}
]
}
@@ -8,6 +8,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
@@ -191,8 +192,33 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
}
}
// keptRead is a state the tools bundle reads, as the index package reads a store: Get and Keys; it writes nothing.
type keptRead struct{ s stdio.KeptState }
func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
e, err := k.s.Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
func (keptRead) Delete(context.Context, string, uint64) error { return errReadOnly }
var errReadOnly = errors.New("the tools bundle only reads this state")
func main() {
if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)); err != nil {
ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...)
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
@@ -0,0 +1,32 @@
package main
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository
// and merge, answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
// invokes, which is the grant to call it.
const mergedTool = delivery.ForgeBridgeModule + ".bridge_merged_message"
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
// restart, a new build). An error is the bridge's own words.
func messageVia(call toolCaller) func(ctx context.Context, repository, merge string) ([]byte, error) {
return func(_ context.Context, repository, merge string) ([]byte, error) {
raw, err := call(mergedTool, map[string]string{"repository": repository, "merge": merge})
if err != nil {
return nil, err
}
var a struct {
Message json.RawMessage `json:"message"`
}
if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 {
return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err)
}
return a.Message, nil
}
}
@@ -9,7 +9,10 @@ import (
"io"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -17,9 +20,15 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-indexer/internal/index"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -42,22 +51,89 @@ accounts {
// bus with the stream FORGE and the indexer's durable <node>_<module> on it, the buckets the controller makes from the
// manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and
// no mesh-delivery-gate_source bucket.
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) {
//
// And the take-in step (ADR 0633, issue 1585 I8a, I8b): the indexer's states pending, verdicts, graph, taken and packages
// and its stream take-in are there, mesh-build-go reported its toolchain, and the node-engine's composition answers the
// build of the indexer that runs. A merge changing the indexer's own tree writes only the indexer's entry, its manifest,
// its verdict and its key of the graph (and the packages it publishes); the rest is pending, said take-in.pending and
// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as
// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end
// then make one take-in, said take-in.taken under the group's key.
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
var running atomic.Pointer[string]
first := "fp-the-indexer-that-runs"
running.Store(&first)
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
})
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
for _, f := range []delivery.Forge{
{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
} {
data, err := delivery.EncodeForge(f)
}, func(ctx context.Context, js jetstream.JetStream) {
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory,
graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
t.Fatal(err)
}
}
// The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes.
for name, subject := range map[string]string{"INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil {
t.Fatal(err)
}
}
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
key, _ := delivery.ToolchainKey("n1", "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
_ = m.Respond(raw)
}
bridge := func(m *nats.Msg) {
var ask struct{ Repository, Merge string }
_ = json.Unmarshal(m.Data, &ask)
data, ok := bridgeMessages.Load(ask.Repository + "@" + ask.Merge)
if !ok {
raw, _ := json.Marshal(map[string]any{"error": "the gate holds no merge of " + ask.Repository + " with that commit"})
_ = m.Respond(raw)
return
}
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data.([]byte))}, "node": "n1"})
_ = m.Respond(raw)
}
for _, subject := range []string{"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>", "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message"} {
if _, err := js.Conn().Subscribe(subject, bridge); err != nil {
t.Fatal(err)
}
}
for _, subject := range []string{"mesh.seat.node-engine.call.composition.n1.>", "mesh.seat.node-engine.tool.composition.n1"} {
if _, err := js.Conn().Subscribe(subject, answer); err != nil {
t.Fatal(err)
}
}
})
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
for _, f := range []forgemsg.Forge{
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
} {
data, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil {
t.Fatal(err)
}
@@ -83,11 +159,202 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want)
}
}
// I8a: pull request 12 merged, changing the indexer's own tree and a's.
manifest, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
const ix = delivery.IndexerModule
m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1,
Merge: "1111111111111111111111111111111111111111", Root: "r0",
Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"},
Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`},
Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}}
merged := func(f forgemsg.Forge) uint64 {
t.Helper()
data, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgeMerged)
bridgeMessages.Store(f.Repository+"@"+f.Merge, data)
ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data)
if err != nil {
t.Fatal(err)
}
return ack.Sequence
}
m1Key := takein.PendingKey(merged(m1))
events, err := js.Stream(ctx, "INDEXER_EVENTS")
if err != nil {
t.Fatal(err)
}
// saidAll is every take-in event of kind said under key so far, oldest first.
saidAll := func(kind, key string) []takein.TakeIn {
event, _ := takein.TakeInEvent(kind)
info, err := events.Info(ctx)
if err != nil {
t.Fatal(err)
}
var out []takein.TakeIn
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ {
msg, err := events.GetMsg(ctx, seq)
if err != nil || msg.Subject != "mesh.mod."+ix+".event."+event {
continue
}
if ti, err := takein.DecodeTakeIn(msg.Data); err == nil && ti.Key == key {
out = append(out, ti)
}
}
return out
}
// said is the last take-in event under kind and key, within 20 s.
said := func(kind, key string) (takein.TakeIn, bool) {
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if all := saidAll(kind, key); len(all) > 0 {
return all[len(all)-1], true
}
}
return takein.TakeIn{}, false
}
held, ok := said(takein.TakeInPending, m1Key)
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
}
named := held.WaitsFor
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
t.Fatal(err)
}
e, err := kv.Get(ctx, key)
if err != nil {
return nil, false
}
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(delivery.ModulesBucket, ix)
own, err := delivery.DecodeModule(raw)
if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
}
if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
}
if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok {
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() {
t.Errorf("the indexer's key %s of the module graph is not written", gk)
}
repoKey, _ := delivery.RepositoryKey(m1.Repository)
if _, ok := get(delivery.ModulesBucket, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(delivery.TakenBucket, repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
{Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}}
g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head,
Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"},
Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members}
g2 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head,
Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"},
Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members}
g1Key, groupKey := takein.PendingKey(merged(g1)), takein.GroupTakeInKey("feat/g")
if gathered, ok := said(takein.TakeInPending, groupKey); !ok || gathered.What != takein.TakeInGroup || len(gathered.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok)
}
// indexer_pending, the tool over the same state, names both waits.
pendingKV, err := js.KeyValue(ctx, index.PendingBucket)
if err != nil {
t.Fatal(err)
}
waits := func() []takein.PendingWait {
t.Helper()
answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
return answer.(takein.PendingAnswer).Waits
}
var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
break
}
}
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(delivery.TakenBucket, repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(delivery.ModulesBucket, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
}
// I8b: the group's second member and its end: one take-in of both, said under the group's key.
merged(g2)
end, err := forgemsg.EncodeForgeGroupEnd(forgemsg.ForgeGroupEnd{Group: "feat/g", Members: []forgemsg.GroupMerge{
{Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+forgemsg.ForgeGroupEndEvent, end); err != nil {
t.Fatal(err)
}
if taken, ok := said(takein.TakeInTaken, groupKey); !ok || taken.What != takein.TakeInGroup || len(taken.Merges) != 2 {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok)
}
var hasC, hasK bool
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
_, hasC = get(delivery.ModulesBucket, "c")
_, hasK = get(delivery.ModulesBucket, "k")
w = waits()
}
if !hasC || !hasK || len(w) != 0 {
t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w)
}
// One take-in: neither member was said as a revision of its own.
if len(saidAll(takein.TakeInPending, g1Key)) != 0 {
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
}
}
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
@@ -166,11 +433,24 @@ func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...f
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
var stop func()
start := func() {
runCtx, cancel := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
stop = func() { cancel(); <-done }
}
start()
t.Cleanup(func() { stop() })
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build.
restart := func() { stop(); start() }
return js, ctx, candidates, modules, restart
}
// startIndexer is startIndexerRun for a test that never starts the program again.
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
return js, ctx, candidates, modules
}
@@ -202,7 +482,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
})
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}})
if err != nil {
@@ -256,7 +536,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
t.Fatal(err)
}
})
forge := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
forge := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -264,7 +544,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
for name, raw := range judged {
forge.Manifests["modules/"+name], forge.Trees["modules/"+name] = raw, "tree-"+name
}
data, err := delivery.EncodeForge(forge)
data, err := forgemsg.EncodeForge(forge)
if err != nil {
t.Fatal(err)
}
@@ -358,7 +638,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi")
})
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -445,7 +725,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
t.Fatal(err)
}
})
pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated,
pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated,
Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}})
if err != nil {
t.Fatal(err)
@@ -615,7 +895,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
place(ctx, t, "n1", "mesh-build-go")
place(ctx, t, "n2")
})
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -24,6 +24,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -153,6 +154,34 @@ type delivered interface {
Metadata() (*jetstream.MsgMetadata, error)
}
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
// repository named forge/group, whose merged message ends the same way, is never taken for one.
var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) {
end, err := forgemsg.DecodeForgeGroupEnd(m.Data())
if err != nil {
logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
_ = m.Term()
return
}
h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group
applied, words, err := ix.EndGroup(ctx, end)
h.Written, h.Late = applied.Written, applied.Late
if err != nil {
logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err)
h.Outcome, h.Words = index.HeardLeft, err.Error()
return
}
h.Outcome, h.Words = index.HeardApplied, words
if err := m.DoubleAck(ctx); err != nil {
logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err)
}
}
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
@@ -171,7 +200,12 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
}
}()
f, err := delivery.DecodeForge(m.Data())
// A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode.
if m.Subject() == groupEndSubject {
handleGroupEnd(ctx, m, ix, &h, logger)
return
}
f, err := forgemsg.DecodeForge(m.Data())
if err != nil {
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
@@ -182,7 +216,13 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
if f.Merge != "" {
h.Commit = f.Merge
}
applied, err := ix.ApplyMessage(ctx, f)
var applied index.Applied
if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 {
// A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then.
applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f)
} else {
applied, err = ix.ApplyMessage(ctx, f)
}
h.Written, h.Late = applied.Written, applied.Late
switch {
case errors.Is(err, index.ErrUnreadable):
@@ -303,12 +343,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf)
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}}
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred)}
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
// revisions in with its old judge.
ix.FixOwnBuild(ctx)
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
@@ -368,7 +418,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if !gate.Toolchain(module, key, value, deleted) {
return nil
}
return ix.Recompute(ctx)
// The modules written again, and pending evaluated again (ADR 0633 §1): the build a revision names moves with
// the toolchain.
return ix.ToolchainReported(ctx)
},
recompute: ix.Recompute}, logger)
if err := tools.start(); err != nil {
@@ -448,11 +500,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer tc.Stop()
}
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
default:
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
// The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event
// said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way.
pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream)
if perr != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr)
}
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
// starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in
// (indexer_take_in_pending) is answered at any time: the takeins watch runs already.
if _, words, err := ix.TakeInAtStart(ctx); err != nil {
return fmt.Errorf("the take-in at start: %w", err)
} else if words != "" {
logger.Printf("take-in at start: %s", words)
}
// The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a
// try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates.
// A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same.
_ = ix.RecoverTries(ctx)
if perr == nil {
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf)
if err != nil {
return err
}
@@ -11,6 +11,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
@@ -73,8 +74,8 @@ const head = "0123456789abcdef0123456789abcdef01234567"
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
st := &memStore{vals: map[string][]byte{}}
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
heard := &memStore{vals: map[string][]byte{}}
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
@@ -84,7 +85,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
}
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
// repository, commit and what it did.
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull ||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
t.Errorf("heard = %+v", h)
}
@@ -102,6 +103,31 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
t.Errorf("heard = %+v", h)
}
// 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked
// (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end.
for _, c := range []struct {
data string
termd int
logged string
}{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""},
{`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} {
logs.Reset()
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)}
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) {
t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String())
}
if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 {
t.Errorf("heard of a group's end = %+v", h)
}
}
// The repository forge/group's merged message ends as a group's end does: it is a forge message all the same.
logs.Reset()
m = &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)}
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
if !strings.Contains(logs.String(), "not a forge message") {
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
}
}
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
@@ -9,6 +9,8 @@ import (
"path"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
@@ -18,7 +20,7 @@ import (
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
// forgemsg.Forge.Paths; novox/hq ADR 0499 clause 3).
const commitTreesTool = "gitea.gitea_commit_trees"
// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
@@ -100,7 +102,7 @@ func filesVia(call toolCaller) func(ctx context.Context, repository, sha string,
}
for _, d := range dirs {
if _, ok := a.Files[d]; !ok {
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
return nil, fmt.Errorf("the answer of %s lists no files of %q: %w", commitTreesTool, d, index.ErrNotInTree)
}
}
return a.Files, nil
@@ -119,6 +121,12 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
return goFileVia(call, "go.mod")
}
// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is
// none. The seat definitions of a revision are read with it (seatsof.go).
func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) {
return goFileVia(call, "")
}
// goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does
// not hold is none.
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
@@ -128,7 +136,10 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
p := name
if dir != "" && dir != delivery.RootPath {
switch {
case name == "": // readFileVia: dir is the file's whole path
p = dir
case dir != "" && dir != delivery.RootPath:
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
@@ -258,8 +269,8 @@ const (
)
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
return func(context.Context) ([]delivery.OpenPull, error) {
func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) {
return func(context.Context) ([]forgemsg.OpenPull, error) {
raw, err := call(openPullsTool, map[string]any{})
if err != nil {
return nil, err
@@ -275,12 +286,40 @@ func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, er
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
}
out := []delivery.OpenPull{}
out := []forgemsg.OpenPull{}
for _, p := range *o.Pulls {
if p.Base == trunkBranch && p.Number > 0 {
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
out = append(out, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
}
}
return out, nil
}
}
// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries).
const openTriesTool = "mesh-delivery-checker.checker_tries"
// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596).
func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) {
return func(context.Context) ([]tryevent.Open, error) {
raw, err := call(openTriesTool, map[string]any{})
if err != nil {
return nil, err
}
// No try open is an answer: the checker's list is empty, or null in an older checker (issue 1921). An answer without the
// key is none.
var o map[string]json.RawMessage
if err := json.Unmarshal(raw, &o); err != nil {
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
}
list, ok := o["tries"]
if !ok {
return nil, fmt.Errorf("the answer of %s is not {tries}: no tries key", openTriesTool)
}
var tries []tryevent.Open
if err := json.Unmarshal(list, &tries); err != nil {
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
}
return tries, nil
}
}
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"mesh-delivery-indexer/internal/index"
"slices"
"strings"
"testing"
@@ -50,6 +51,15 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
if _, has := gotArgs.(map[string]any)["paths"]; has {
t.Error("the trunk asked for paths")
}
// A directory the answer lists no files of is a failure of the tree (index.ErrNotInTree); a call that fails is not.
noDir := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"files":{}}`), nil }
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
}
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
}
// The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine.
var asked string
own := func(key string, body any) (json.RawMessage, error) {
@@ -59,10 +69,45 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" {
t.Errorf("own build = %q %v asked %q", fp, err, asked)
}
// Issue 1921: the start during a replacement. What runs still names the old build, the want (composed by the node itself)
// the successor, which this process is: it fixes the successor. A want that is not self, or is removed, leaves what runs.
for _, row := range []struct{ name, answer, want string }{
{"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
{"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"},
{"replacing: the apply of the want failed or is held", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:new","held":true}}`, "src2:old"},
{"replacing: an attempt of another build", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:older"}}`, "src2:new"},
{"the want already runs", `{"runs":{"fp":"src2:new"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
{"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"},
{"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"},
} {
ask := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
if fp, err := ownBuildVia(ask, "n1")(context.Background()); err != nil || fp != row.want {
t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want)
}
}
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil {
t.Error("an engine that says nothing runs gave a build")
}
// Issue 1921: the checker's list of open tries; none open is an answer, whether [] or null.
for _, row := range []struct {
name, answer string
want int
refused bool
}{
{"one open", `{"tries":[{"id":"try-1","state":"running","whole":true,"members":[{"repository":"novox/a","head":"h","trunk":"t"}]}]}`, 1, false},
{"an empty list", `{"tries":[]}`, 0, false},
{"null, as an older checker answers none", `{"tries":null}`, 0, false},
{"no tries key", `{}`, 0, true},
{"not json", `nope`, 0, true},
{"tries of another shape", `{"tries":{"a":1}}`, 0, true},
} {
call := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
got, err := openTriesVia(call)(context.Background())
if (err != nil) != row.refused || len(got) != row.want {
t.Errorf("%s: %d tries, error %v; want %d, refused %v", row.name, len(got), err, row.want, row.refused)
}
}
}
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
@@ -25,10 +25,32 @@ func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string
Runs *struct {
FP string `json:"fp"`
} `json:"runs"`
Want *struct {
FP string `json:"fp"`
Self bool `json:"self"`
Removed bool `json:"removed"`
} `json:"want"`
// Attempt is the engine's last apply that did not end running (failed, or held), with the build it was of.
Attempt *struct {
FP string `json:"fp"`
} `json:"attempt"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err)
}
// While the engine's apply of the want is in progress (what runs still names the old build, the want names another, and no
// attempt names that want as failed or held), this process is the successor and fixes the want's build. self is the node's
// standing way of composing the module, not a mark of this process, so it is not enough alone: an old process started again
// while the want names a successor that does not run (its apply failed or is held) takes what runs. The start fixes the
// build once (quick fix, ADR 0639 §7: nothing is learnt later, nothing waits on a timer); the thorough form is the
// engine passing each process the fingerprint it starts it with.
running := ""
if a.Runs != nil {
running = a.Runs.FP
}
if w := a.Want; w != nil && w.Self && !w.Removed && w.FP != "" && w.FP != running && (a.Attempt == nil || a.Attempt.FP != w.FP) {
return w.FP, nil
}
if a.Runs == nil || a.Runs.FP == "" {
return "", errors.New("the engine says no fingerprint of the indexer runs")
}
@@ -13,6 +13,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -71,10 +72,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
}
serveForge(t, js.Conn(), listing, goSumContent)
})
merged := func(repo string, number int, merge string, f delivery.Forge) {
merged := func(repo string, number int, merge string, f forgemsg.Forge) {
t.Helper()
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = delivery.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
data, err := delivery.EncodeForge(f)
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = forgemsg.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
data, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
@@ -101,10 +102,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
"tsconfig.json": "5555555555555555555555555555555555555555"}}
}
// mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself.
merged("novox/mesh-sdk", 70, m1, delivery.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")})
module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 })
merged("novox/mesh-catalog", 301, catalogMerge, delivery.Forge{Root: "r0",
merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0",
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}})
@@ -140,11 +141,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
// over its named tree, is not written (issue 1758, ADR 0499 clause 6; the operator, 2026-10-25: a module's value moves
// only with the module).
const goOnly = "4444444444444444444444444444444444444444"
merged("novox/mesh-sdk", 72, goOnly, delivery.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
merged("novox/mesh-sdk", 72, goOnly, forgemsg.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
Trees: map[string]string{"": "cccccccccccccccccccccccccccccccccccccccc"}, Manifests: map[string]string{"": string(sdkManifest)},
Paths: sdkObjects("6666666666666666666666666666666666666666")})
// mesh-sdk merges a change of buildstep alone (its version moves: src changed too).
merged("novox/mesh-sdk", 71, m2, delivery.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")})
sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 })
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) {
@@ -11,6 +11,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
@@ -73,7 +74,10 @@ func (w *previewWatch) ensure() error {
}
})
}()
return w.ix.RecomputeCandidates(w.ctx)
if err := w.ix.RecomputeCandidates(w.ctx); err != nil {
return err
}
return w.ix.WriteTryCandidates(w.ctx)
}
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
@@ -90,3 +94,44 @@ func (w *previewWatch) Written(m delivered) {
}
_ = m.Ack()
}
// tryEventSubject is the subject of an event of the checker's stream checks about a try.
func tryEventSubject(event string) string {
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from
// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and
// the server delivers it again; a body that is no try event is terminated.
func (w *previewWatch) heardChecks(m delivered) {
var event string
switch m.Subject() {
case tryEventSubject(tryevent.Started):
event = tryevent.Started
case tryEventSubject(tryevent.Ended):
event = tryevent.Ended
default:
w.Written(m)
return
}
e, err := tryevent.Decode(event, m.Data())
if err != nil {
w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
if event == tryevent.Started {
var ms []index.TryMember
for _, x := range e.Members {
ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk})
}
err = w.ix.StartTry(w.ctx, e.ID, ms)
} else {
err = w.ix.EndTry(w.ctx, e.ID)
}
if err != nil {
w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err)
return
}
_ = m.Ack()
}
@@ -16,6 +16,11 @@ type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
TakeInRepository(ctx context.Context, repository string) index.TakenIn
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
DropPending(ctx context.Context, key, by, why string) index.TakenIn
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -34,7 +39,22 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
return
}
asked, r := "take-in", index.TakenIn{}
if a.Repository != "" {
if a.RecoverTries {
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
a.Module = "tries"
} else if a.TryCandidate != "" && a.Drop {
asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
a.Module = a.TryCandidate
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" {
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Repository != "" {
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
a.Module = a.Repository
} else if a.Judge {
@@ -6,6 +6,7 @@ import (
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
@@ -62,3 +63,28 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string,
}
}
}
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
return func(ctx context.Context, t takein.TakeIn) error {
event, err := takein.TakeInEvent(t.Kind)
if err != nil {
return err
}
data, err := takein.EncodeTakeIn(t)
if err != nil {
return err
}
msg := nats.NewMsg(cred.EventSubject(event))
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
_, err = js.PublishMsg(pub, msg)
return err
}
}
@@ -46,7 +46,19 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
v, err := judge.Request(r)
// The request's seat definitions (ADR 0636 §4): the composer's request type of this build does not decode them.
var seats struct {
Seats []json.RawMessage `json:"seats"`
}
if err := json.Unmarshal(raw, &seats); err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
files := make([][]byte, len(seats.Seats))
for i, s := range seats.Seats {
files[i] = s
}
v, err := judge.RequestWith(r, files)
if err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
+10 -2
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -24,6 +24,14 @@ require (
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
)
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -7,12 +7,13 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge {
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
if manifest != "" {
f.Manifests = map[string]string{"modules/avahi": manifest}
}
@@ -11,6 +11,9 @@ import (
"sync/atomic"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -80,6 +83,9 @@ type Indexer struct {
// repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their
// content; nil reads none.
GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error)
// ReadFile reads one file of a repository at a commit (gitea.gitea_get_file); a file the commit does not hold is nil.
// With Files it reads the seat definitions a revision carries (seatsof.go); nil reads none.
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
// Now is the clock of a taken value; nil is time.Now.
Now func() time.Time
Builds *Builds
@@ -98,7 +104,7 @@ type Indexer struct {
// ReadOpen asks the forge which pull requests into the trunk are open now (gitea_open_pulls). A catalogue pull message
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error)
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
@@ -106,7 +112,7 @@ type Indexer struct {
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
@@ -115,12 +121,38 @@ type Indexer struct {
filling atomic.Bool // a FillTaken runs in the background (KickFill)
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild)
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
OwnBuild func(ctx context.Context) (string, error)
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
// Message asks the forge bridge for the merged message of a merge, repository and merge commit in, the message whole out (a
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
Message func(ctx context.Context, repository, merge string) ([]byte, error)
heldMu sync.Mutex
held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
// takes every merge in at once, as before the take-in step.
Pending Store
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
// OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go).
OpenTries func(ctx context.Context) ([]tryevent.Open, error)
// TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none.
TryRecovery Store
tryMu sync.Mutex // one try event or recovery at a time (trycandidates.go)
recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go)
tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
}
func (ix *Indexer) logf(format string, args ...any) {
@@ -133,23 +165,23 @@ func (ix *Indexer) logf(format string, args ...any) {
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
// merged message also writes the fingerprint of each module of its repository (modules.go).
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
_, err := ix.ApplyMessage(ctx, f)
return err
}
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, error) {
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
ix.kickJudge(ctx)
var applied Applied
if f.Kind == delivery.ForgeOpen {
if f.Kind == forgemsg.ForgeOpen {
return applied, ix.applyOpen(ctx, f)
}
if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged {
if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged {
return applied, nil
}
if f.Kind == delivery.ForgeMerged && ix.Modules != nil {
if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil {
w, err := ix.applyMerged(ctx, f)
applied = Applied{Written: w.written, Late: w.late}
if err != nil {
@@ -178,7 +210,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
}
var digests map[string]string
var held map[string]delivery.Module
if f.Kind == delivery.ForgePull {
if f.Kind == forgemsg.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
return applied, err
@@ -207,15 +239,26 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err)
}
}
return applied, ix.writeCandidate(ctx, f, digests, held)
}
// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a
// merge a newer merge of the same repository covers (it needs only the repository, number and branch).
func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error {
return ix.writeCandidate(ctx, f, nil, nil)
}
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) error {
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
return applied, nil
return nil
}
key, err := delivery.CandidatesKey(f.Branch)
if err != nil {
return applied, fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
return applied, write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
var cur *delivery.Candidate
if found {
c, err := delivery.DecodeCandidate(value)
@@ -245,8 +288,8 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is
// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request
// opened after the bridge read the forge is heard after this message and added again by its own.
func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
named := make(map[delivery.OpenPull]bool, len(f.Open))
func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
named := make(map[forgemsg.OpenPull]bool, len(f.Open))
for _, p := range f.Open {
named[p] = true
}
@@ -255,6 +298,9 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
return fmt.Errorf("listing the candidates: %w", err)
}
for _, key := range keys {
if IsTryGroup(key) {
continue // a try's candidate is no pull request's: the checker's try owns it (trycandidates.go)
}
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
@@ -264,7 +310,7 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
}
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: p.Number}]
})
switch {
case len(kept) == len(c.Pulls):
@@ -331,14 +377,14 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
var pulls []delivery.CandidatePull
var base uint64
if cur != nil {
pulls, base = slices.Clone(cur.Pulls), cur.Base
}
at := slices.IndexFunc(pulls, func(p delivery.CandidatePull) bool { return p.Repository == f.Repository && p.Number == f.Number })
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
if at < 0 {
return cur
}
@@ -10,6 +10,7 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -27,7 +28,7 @@ var (
// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older
// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1).
func (ix *Indexer) writePredefined(ctx context.Context, f delivery.Forge) error {
func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error {
if ix.Predefined == nil || len(f.Predefined) == 0 {
return nil
}
@@ -203,8 +204,8 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n
// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written,
// a closed one deleted. A head held again without its directories leaves the key as it is.
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, held map[string]delivery.Module) error {
if f.Action == delivery.PullClosed {
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f forgemsg.Forge, held map[string]delivery.Module) error {
if f.Action == forgemsg.PullClosed {
return ix.dropConfigCandidate(ctx, f.Repository, f.Number)
}
if !f.DirsKnown {
@@ -220,7 +221,7 @@ func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge,
// configCandidatesAfterMerge writes again every open pull request's key that names a module the merge changed, since what
// it adds is judged against the trunk (ADR 0565 §2), from the head's manifests its candidate keeps; and deletes the merged
// pull request's.
func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Forge, changed []string) error {
func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Forge, changed []string) error {
if ix.ConfigCandidates == nil {
return nil
}
@@ -243,6 +244,9 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Fo
}
var errs []error
for _, g := range groups {
if IsTryGroup(g) {
continue // a try's candidate holds no pull request's config declarations (trycandidates.go)
}
raw, _, found, err := ix.Store.Get(ctx, g)
if err != nil || !found {
continue
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -77,12 +78,29 @@ func TestTheControlRecords(t *testing.T) {
t.Errorf("a head declaring nothing new = %+v, want an empty set", c)
}
closed := pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40))
closed.Action = delivery.PullClosed
closed.Action = forgemsg.PullClosed
if err := ix.Apply(ctx, closed); err != nil {
t.Fatal(err)
}
if _, ok := cands.vals[key]; ok {
t.Error("a closed pull request's declarations stayed")
}
// A group try's candidate (number 0) holds no pull request's declarations: a merge that touches its module writes none
// for it (fails when configCandidatesAfterMerge no longer skips a try's group).
tryPull := delivery.CandidatePull{Head: delivery.Head{Repository: catalogueRepo, Head: h1}, Modules: []string{"avahi"},
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(trunk)), None2: NoneNoPreview}}}
tryCand := delivery.Candidate{V: delivery.Version, Group: "try-5", Pulls: []delivery.CandidatePull{tryPull}}
tryCand.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{tryPull.Head})
tryCand.FPSet = delivery.FingerprintSet(tryCand.Pulls)
data, err := delivery.EncodeCandidate(tryCand)
if err != nil {
t.Fatal(err)
}
if err := ix.Store.Create(ctx, "try-5", data); err != nil {
t.Fatal(err)
}
if err := ix.configCandidatesAfterMerge(ctx, mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t2"}), []string{"avahi"}); err != nil || len(cands.vals) != 0 {
t.Errorf("a merge touching a try's module wrote %v for it (%v), want none", cands.vals, err)
}
})
}
@@ -2,9 +2,11 @@ package index
import (
"context"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -25,6 +27,16 @@ type memStore struct {
next uint64
writes int
beforeWrite func()
failWrite func(key string) error // a write's own error, asked before each Create and Update
maxValue int // the bucket's value cap (buscompose.StateMaxValueBytes); a larger value is refused as the bus refuses it
}
// tooBig is the bus's refusal of a value over a bucket's cap.
func (m *memStore) tooBig(value []byte) error {
if m.maxValue > 0 && len(value) > m.maxValue {
return errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed")
}
return nil
}
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
@@ -58,18 +70,34 @@ func (m *memStore) put(key string, value []byte) {
}
func (m *memStore) Create(_ context.Context, key string, value []byte) error {
m.hook()
if f := m.failWrite; f != nil {
if err := f(key); err != nil {
return err
}
}
if _, ok := m.vals[key]; ok {
return ErrConflict
}
if err := m.tooBig(value); err != nil {
return err
}
m.writes++
m.put(key, value)
return nil
}
func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint64) error {
m.hook()
if f := m.failWrite; f != nil {
if err := f(key); err != nil {
return err
}
}
if _, ok := m.vals[key]; !ok || m.revs[key] != rev {
return ErrConflict
}
if err := m.tooBig(value); err != nil {
return err
}
m.writes++
m.put(key, value)
return nil
@@ -91,9 +119,9 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte {
return logged(t, "build-"+module, module, repo, path, "", "", at)
}
func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
Action: delivery.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge {
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
}
func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
@@ -212,13 +240,13 @@ func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1
ctx := context.Background()
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
gone := func(repo string, n int, head string) delivery.Forge {
gone := func(repo string, n int, head string) forgemsg.Forge {
if how == "closed" {
f := pull(repo, n, "feat/x", head)
f.Action = delivery.PullClosed
f.Action = forgemsg.PullClosed
return f
}
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
}
if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil {
t.Fatal(err)
@@ -330,8 +358,8 @@ func TestAnOddBranchNameKeysEscaped(t *testing.T) {
}
}
func open(pulls ...delivery.OpenPull) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: pulls}
func open(pulls ...forgemsg.OpenPull) forgemsg.Forge {
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeOpen, Open: pulls}
}
// 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is
@@ -339,7 +367,7 @@ func open(pulls ...delivery.OpenPull) delivery.Forge {
func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/AC7
ix, st, _ := newIndexer(t)
ctx := context.Background()
for _, f := range []delivery.Forge{
for _, f := range []forgemsg.Forge{
pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/a"),
pull("novox/mesh-catalog", 280, "feat/old", h3, "modules/c"),
@@ -351,7 +379,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
before, _ := read(t, st, "feat/x")
// Open: mesh-delivery#12 only (and a pull request no group holds, which changes nothing).
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil {
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil {
t.Fatal(err)
}
c, ok := read(t, st, "feat/x")
@@ -374,7 +402,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
}
// The same open message again changes nothing (no write), and an empty one drops every group.
writes := st.writes
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
t.Fatal(err)
}
if st.writes != writes {
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -48,7 +49,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[string]string, error) {
digests := make(map[string]string, len(f.Manifests))
for dir, raw := range f.Manifests {
module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module
@@ -5,6 +5,7 @@ import (
"os"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -37,7 +38,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt
return ix, mods, manifests
}
func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
func withManifests(f forgemsg.Forge, manifests map[string]string, locks map[string]map[string]string) forgemsg.Forge {
f.Manifests, f.Locks = manifests, locks
return f
}
@@ -81,8 +82,8 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
if err := ix.Apply(context.Background(), f); err != nil {
t.Errorf("a second merge: %v", err)
}
pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
pull := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
if err := ix.Apply(context.Background(), pull); err != nil {
t.Errorf("a pull message: %v", err)
}
@@ -7,6 +7,7 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -31,20 +32,20 @@ import (
//
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
// deletes nothing, and is said; it is asked once, before anything is written.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) {
func (ix *Indexer) applyMerged(ctx context.Context, f forgemsg.Forge) (*mergeWrite, error) {
return ix.applyManifests(ctx, f, true)
}
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
func (ix *Indexer) applyManifests(ctx context.Context, f forgemsg.Forge, drop bool) (*mergeWrite, error) {
return ix.applyMoving(ctx, f, drop, nil)
}
// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
w := newMergeWrite(f)
if drop {
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
@@ -180,7 +181,7 @@ func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m
// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[string]map[string]string, map[string]string) {
objects := map[string]map[string]string{}
need := map[string][]string{}
for dir, raw := range f.Manifests {
@@ -220,7 +221,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
var handed []string
elsewhere := map[string]delivery.Forge{}
elsewhere := map[string]forgemsg.Forge{}
for _, key := range slices.Sorted(maps.Keys(held)) {
cur := held[key]
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
@@ -354,7 +355,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
others := ix.otherPackages(ctx, f, dir, cur, has)
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
return others
@@ -368,7 +369,7 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir stri
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
held := func() []string {
if !has || cur.Inputs2 == nil {
return nil
@@ -395,7 +396,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir st
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
// from the forge (GoSum).
func (ix *Indexer) otherPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
if carried := f.Packages[dir]; len(carried) > 0 {
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
}
@@ -10,6 +10,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -71,12 +72,12 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2
// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its
// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that
// holds a module.json).
func mergedMsg(repo string, trees map[string]string) delivery.Forge {
func mergedMsg(repo string, trees map[string]string) forgemsg.Forge {
manifests := map[string]string{}
for dir := range trees {
manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}`
}
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
}
func nameOfDir(dir string) string {
@@ -459,7 +460,7 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
gone.Trees, gone.Manifests = nil, nil
for _, tc := range []struct {
name string
merges []delivery.Forge
merges []forgemsg.Forge
repository string // modules.a's repository at the end, "" for none
deletedKeyA bool
// Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge
@@ -472,20 +473,20 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
stored bool
bGoneAtEnd bool
}{
{name: "named while its trunk holds it: left", merges: []delivery.Forge{held, named}, repository: from},
{name: "named, then dropped by its trunk: handed over in place", merges: []delivery.Forge{held, named, dropped}, repository: to},
{name: "dropped by its trunk, then named: deleted, then written new", merges: []delivery.Forge{held, dropped, named}, repository: to, deletedKeyA: true},
{name: "dropped by its trunk with no merge naming it: deleted", merges: []delivery.Forge{held, dropped}, deletedKeyA: true},
{name: "named while its trunk holds it: left", merges: []forgemsg.Forge{held, named}, repository: from},
{name: "named, then dropped by its trunk: handed over in place", merges: []forgemsg.Forge{held, named, dropped}, repository: to},
{name: "dropped by its trunk, then named: deleted, then written new", merges: []forgemsg.Forge{held, dropped, named}, repository: to, deletedKeyA: true},
{name: "dropped by its trunk with no merge naming it: deleted", merges: []forgemsg.Forge{held, dropped}, deletedKeyA: true},
{name: "no module.json left, another trunk holds it, nothing waits in memory: written from that trunk, never deleted",
merges: []delivery.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
merges: []forgemsg.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
{name: "no module.json left, no other trunk holds it: deleted",
merges: []delivery.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
merges: []forgemsg.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
{name: "taken in by repository, a head with no module.json: handed over from the other trunk",
merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
{name: "taken in by repository, a head with modules: answered by name",
merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
{name: "taken in by repository, no head named: answered so",
merges: []delivery.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
merges: []forgemsg.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
} {
t.Run(tc.name, func(t *testing.T) {
mods := &deletesSeen{memStore: newMem()}
@@ -7,6 +7,7 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -14,9 +15,9 @@ import (
// holds it at the head the latest source names: the module directories there are asked of the forge (Trunk) and the manifest of
// each directory named like the module is read (Read). It answers the merge to hand the module over with, or none: a forge
// that cannot say leaves the module to be dropped as it was, said (issue 1367).
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (delivery.Forge, bool) {
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (forgemsg.Forge, bool) {
if ix.Trunk == nil || ix.Read == nil {
return delivery.Forge{}, false
return forgemsg.Forge{}, false
}
ix.mu.Lock()
heads := maps.Clone(ix.source.heads)
@@ -45,7 +46,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
if read.Trees[dir] == "" || manifestModule(raw) != module {
continue
}
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
@@ -57,7 +58,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
return f, true
}
}
return delivery.Forge{}, false
return forgemsg.Forge{}, false
}
// nameOfDirectory is the last element of a module directory.
@@ -76,8 +77,8 @@ func nameOfDirectory(dir string) string { return dir[strings.LastIndex(dir, "/")
// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
// repository. A later merge of the same repository replaces it.
func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) {
one := forgemsg.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
if l := f.Locks[dir]; len(l) > 0 {
one.Locks = map[string]map[string]string{dir: l}
@@ -91,7 +92,7 @@ func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
ix.mu.Lock()
defer ix.mu.Unlock()
if ix.moving == nil {
ix.moving = map[string]delivery.Forge{}
ix.moving = map[string]forgemsg.Forge{}
}
ix.moving[key] = one
}
@@ -117,7 +118,7 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error {
}
// handOverFrom writes the module at key from the merge f, which names it, in place.
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f delivery.Forge) error {
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error {
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
if err != nil {
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
@@ -7,6 +7,7 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -46,7 +47,7 @@ func (w *mergeWrite) taken(digest string) {
w.fresh[digest] = true
}
func newMergeWrite(f delivery.Forge) *mergeWrite {
func newMergeWrite(f forgemsg.Forge) *mergeWrite {
return &mergeWrite{repository: f.Repository, merge: f.Merge}
}
@@ -0,0 +1,110 @@
package index
import (
"context"
"maps"
"slices"
"sync"
)
// overlayStore is a store whose writes stay in memory over the store it reads through (novox/hq ADR 0633 §1): a merge
// applied to it is computed whole, its every fingerprint and verdict, and nothing reaches the bus. A key it wrote reads
// as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set holds against
// what it reads: a key it wrote has a revision of its own, counted above every revision under it. A nil store under
// it holds nothing.
type overlayStore struct {
under Store
mu sync.Mutex
vals map[string]overlaid
next uint64
}
type overlaid struct {
value []byte
revision uint64
deleted bool
}
func newOverlay(under Store) *overlayStore {
return &overlayStore{under: under, vals: map[string]overlaid{}, next: 1 << 62}
}
func (o *overlayStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
o.mu.Lock()
v, ok := o.vals[key]
o.mu.Unlock()
if ok {
if v.deleted {
return nil, v.revision, false, nil
}
return slices.Clone(v.value), v.revision, true, nil
}
if o.under == nil {
return nil, 0, false, nil
}
return o.under.Get(ctx, key)
}
// put checks the revision read against what the key holds now and writes it.
func (o *overlayStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error {
_, rev, found, err := o.Get(ctx, key)
if err != nil {
return err
}
o.mu.Lock()
defer o.mu.Unlock()
switch {
case create && found, !create && (!found || rev != revision):
return ErrConflict
}
o.next++
o.vals[key] = overlaid{value: slices.Clone(value), revision: o.next, deleted: del}
return nil
}
func (o *overlayStore) Create(ctx context.Context, key string, value []byte) error {
return o.put(ctx, key, value, 0, true, false)
}
func (o *overlayStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
return o.put(ctx, key, value, revision, false, false)
}
func (o *overlayStore) Delete(ctx context.Context, key string, revision uint64) error {
return o.put(ctx, key, nil, revision, false, true)
}
func (o *overlayStore) Keys(ctx context.Context) ([]string, error) {
var keys []string
if o.under != nil {
var err error
if keys, err = o.under.Keys(ctx); err != nil {
return nil, err
}
}
o.mu.Lock()
defer o.mu.Unlock()
keys = slices.DeleteFunc(keys, func(k string) bool { _, ok := o.vals[k]; return ok })
for k, v := range o.vals {
if !v.deleted {
keys = append(keys, k)
}
}
slices.Sort(keys)
return keys, nil
}
// written is what the overlay holds of key: its value, and whether it wrote or deleted it at all.
func (o *overlayStore) written(key string) (value []byte, deleted, ok bool) {
o.mu.Lock()
defer o.mu.Unlock()
v, ok := o.vals[key]
return slices.Clone(v.value), v.deleted, ok
}
// changed is every key the overlay wrote or deleted, in order.
func (o *overlayStore) changed() []string {
o.mu.Lock()
defer o.mu.Unlock()
return slices.Sorted(maps.Keys(o.vals))
}
@@ -7,6 +7,7 @@ import (
"path"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -70,7 +71,7 @@ func moduleAt(held map[string]delivery.Module, repository, dir string) (string,
// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else
// the one the modules bucket holds at that directory of the repository, else the directory's last element (the
// catalogue's rule for nearly every manifest).
func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string {
func moduleOfDir(held map[string]delivery.Module, f forgemsg.Forge, dir string) string {
if name := manifestModule(f.Manifests[dir]); name != "" {
return name
}
@@ -2,6 +2,7 @@ package index
import (
"context"
"errors"
"fmt"
"maps"
"path"
@@ -297,6 +298,12 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
ix.mu.Lock()
got, ok := ix.atHeads[at]
ix.mu.Unlock()
ix.mu.Lock()
failed := ix.failedAt[repo] == head
ix.mu.Unlock()
if failed {
continue // said once for this head; the candidate takes the published version until the head changes
}
if !ok {
var ms []string
ms = append(ms, modules[repo]...)
@@ -307,10 +314,29 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
}
var err error
if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil {
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
// A failure of the tree itself (the directory is not in the tree at that head) no retry changes: it is asked once per
// head. Any other failure may be transient: it is asked again at the next pass, and said once per head.
ix.mu.Lock()
said := ix.saidAt[repo] == head
if ix.saidAt == nil {
ix.saidAt = map[string]string{}
}
ix.saidAt[repo] = head
if errors.Is(err, ErrNotInTree) {
if ix.failedAt == nil {
ix.failedAt = map[string]string{}
}
ix.failedAt[repo] = head
}
ix.mu.Unlock()
if !said {
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
}
continue
}
ix.mu.Lock()
delete(ix.failedAt, repo)
delete(ix.saidAt, repo)
if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
}
@@ -322,5 +348,9 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
return out
}
// ErrNotInTree is a failure of the tree itself: a directory asked of the forge is not in the tree at that commit. No retry
// changes it, so a head that fails so is asked once (candidatePublished).
var ErrNotInTree = errors.New("the directory is not in the tree at that commit")
// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again.
const maxHeadsKept = 64
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,651 @@
package index
import (
"context"
"encoding/json"
"errors"
"reflect"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its
// pending state and the take-in events it says; base is the catalogue merge it was built from.
type takeInBed struct {
ix *Indexer
mods *memStore
pending *memStore
said []takein.TakeIn
bridge map[string][]byte // what the forge bridge answers (bridge_merged_message), by repository@merge
base forgemsg.Forge
logs *strings.Builder
}
func newTakeInBed(t *testing.T) *takeInBed {
t.Helper()
ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")},
mod{delivery.IndexerModule, renamed(t, "avahi", delivery.IndexerModule)})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
b := &takeInBed{ix: ix, mods: mods, pending: newMem(), bridge: map[string][]byte{}, base: f, logs: &strings.Builder{}}
ix.Verdicts = newMem()
b.pending.maxValue = buscompose.StateMaxValueBytes
ix.Pending = b.pending
ix.Message = func(_ context.Context, repository, merge string) ([]byte, error) {
if data, ok := b.bridge[repository+"@"+merge]; ok {
return data, nil
}
return nil, errors.New("the gate holds no merge of " + repository + " with that commit")
}
ix.SayTakeIn = func(_ context.Context, t takein.TakeIn) error { b.said = append(b.said, t); return nil }
logf := ix.Logf
ix.Logf = func(format string, args ...any) {
b.logs.WriteString(format + "\n")
if logf != nil {
logf(format, args...)
}
}
own := b.fp(t, delivery.IndexerModule)
if own == "" || b.fp(t, "avahi") == "" {
t.Fatalf("the bed's modules have no fingerprint")
}
ix.OwnBuild = func(context.Context) (string, error) { return own, nil }
ix.OwnBuildState = newMem()
ix.FixOwnBuild(context.Background()) // fixed once, at the start
return b
}
func (b *takeInBed) fp(t *testing.T, module string) string {
t.Helper()
m, _ := moduleOf(t, b.mods, module)
return m.FP2
}
func (b *takeInBed) commit(t *testing.T, module string) string {
t.Helper()
m, _ := moduleOf(t, b.mods, module)
return m.Commit
}
// merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's).
func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
f := b.base
f.Number, f.Merge, f.Head = number, c, c
f.Trees = map[string]string{}
for dir, tree := range b.base.Trees {
f.Trees[dir] = tree
}
if avahi != "" {
f.Trees["modules/avahi"] = avahi
}
if indexer != "" {
f.Trees["modules/"+delivery.IndexerModule] = indexer
}
return f
}
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
t.Helper()
b.publish(t, f)
if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil {
t.Fatalf("arrive %d: %v", seq, err)
}
}
// publish lets the bed's forge bridge answer the merged message f, as bridge_merged_message does.
func (b *takeInBed) publish(t *testing.T, f forgemsg.Forge) {
t.Helper()
data, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
b.bridge[f.Repository+"@"+f.Merge] = data
}
func (b *takeInBed) keys() []string {
k, _ := b.pending.Keys(context.Background())
slices.Sort(k)
return k
}
func (b *takeInBed) events() []string {
var out []string
for _, s := range b.said {
out = append(out, s.Kind+" "+s.Key)
}
return out
}
func sha(c byte) string { return strings.Repeat(string(c), 40) }
func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
ctx := context.Background()
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
}
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
// build is unknown, from the own-build state.
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
}
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
}
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
}
})
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
b := newTakeInBed(t)
own, avahi0 := b.fp(t, delivery.IndexerModule), b.fp(t, "avahi")
// A merge with no change to the indexer and nothing pending is taken in alone, as before.
b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 {
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in", avahi1, avahi0, b.keys(), b.events())
}
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
}
if g := graphOf(t, b.ix, delivery.IndexerModule); g.FP != named {
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
}
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
}
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
}
// T5: the merge delivered again writes no second key and says nothing again.
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
if len(b.keys()) != 1 || len(b.said) != 1 {
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
}
// A later merge joins: the newest governs and still names the new indexer.
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
}
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+delivery.IndexerModule]))
if len(b.keys()) != 0 || b.fp(t, delivery.IndexerModule) != own || b.commit(t, "avahi") != sha('e') {
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, delivery.IndexerModule), b.commit(t, "avahi"))
}
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
}
})
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
// A successor's manifest invokes a verb of the node-engine's seat that the built-in list lacks and the revision's
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
// accepted with them, and the revision waits for the build it names.
const verb, contracts = "manifest-reads", "modules/seat-contracts"
for _, row := range []struct {
name string
defined bool
refused bool
}{{"definitions the revision does not carry", false, true}, {"definitions the revision carries", true, false}} {
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Manifests = map[string]string{}
for dir, raw := range b.base.Manifests {
f.Manifests[dir] = raw
}
dir := "modules/" + delivery.IndexerModule
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+delivery.IndexerModule+`"`,
`"module": "`+delivery.IndexerModule+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
if row.defined {
f.Trees[contracts] = "tree-seat-contracts"
f.Locks = map[string]map[string]string{}
for d, l := range b.base.Locks {
f.Locks[d] = l
}
f.Locks[contracts] = map[string]string{}
f.Manifests[contracts] = `{"module":"seat-contracts","version":"1","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
}
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case contracts + "/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
case contracts + "/parts/reads.json":
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"` + verb + `"}]}`), nil
}
return nil, nil
}
}
b.arrive(t, 2, f)
waits := PendingWaits(readPending(t, b))
refused := ""
if len(b.said) > 0 {
refused = b.said[0].Refused
}
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
t.Errorf("%s: the seats are not put back after the dry run", row.name)
}
}
})
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
g := b.merge(30, sha('f'), "", "")
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g)
named := b.fp(t, delivery.IndexerModule)
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
mk := func(ask func(context.Context) (string, error)) *Indexer {
n := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum}
n.toolchains = b.ix.toolchains
n.OwnBuild = ask
n.FixOwnBuild(ctx)
return n
}
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
// wait was said before; and indexer_pending names the known build and since when.
oldBuild, _ := b.ix.OwnBuild(ctx)
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
b.logs.Reset()
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
}
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
t.Errorf("a hand ask while the own build is unknown: %+v", r)
}
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return named, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
b := newTakeInBed(t)
b.ix.Taken = newMem()
// The commits in the order they were made: one is in another when it comes first.
order := "a756bc"
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil
}
member := func(repo string, number int, c string) forgemsg.Forge {
f := b.merge(number, c, "", "")
f.Group = "feat/take-in"
f.Members = []delivery.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}}
if repo != catalogueRepo {
f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil
}
f.Head = map[int]string{40: sha('1'), 41: sha('2')}[number]
return f
}
b.arrive(t, 1, member(catalogueRepo, 40, sha('7')))
if b.commit(t, "avahi") == sha('7') || !slices.Equal(b.events(), []string{"pending g.feat/take-in"}) {
t.Fatalf("first member: avahi at %s, said %v: want held, the group said pending", b.commit(t, "avahi"), b.events())
}
// A merge in another repository is taken in alone; one in a member's repository waits behind the group.
alone := b.merge(50, sha('5'), "", "")
alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil
b.arrive(t, 2, alone)
b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", ""))
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1), takein.PendingKey(3)}) {
t.Fatalf("pending %v: want the member and the merge behind it", b.keys())
}
end := forgemsg.ForgeGroupEnd{Group: "feat/take-in", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}}
if _, _, err := b.ix.EndGroup(ctx, end); err == nil {
t.Fatal("an end before its last member is taken in")
}
b.arrive(t, 4, member("novox/mesh-sdk", 41, sha('b')))
if _, _, err := b.ix.EndGroup(ctx, end); err != nil {
t.Fatal(err)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') {
t.Fatalf("end: pending %v, avahi at %s: want the group and the merge behind it taken in", b.keys(), b.commit(t, "avahi"))
}
if want := []string{"pending g.feat/take-in", "pending g.feat/take-in", "taken g.feat/take-in"}; !slices.Equal(b.events(), want) {
t.Errorf("said %v, want %v", b.events(), want)
}
// R1: the end published again after the take-in changes nothing and is not delivered again.
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
t.Errorf("an end again: %q %v", words, err)
}
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
b.said = nil
g := member(catalogueRepo, 40, sha('c'))
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
g.Members = g.Members[:1]
b.arrive(t, 9, g)
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
t.Fatal(err)
}
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
}
})
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
b.ix.ownMu.Lock()
b.ix.ownFP = named
b.ix.ownMu.Unlock()
// The other indexer claims the merge between this one's read and its claim.
b.pending.beforeWrite = func() {
v := b.pending.vals[takein.PendingKey(2)]
var p Pending
_ = json.Unmarshal(v, &p)
p.Taking = "the other indexer"
data, _ := json.Marshal(p)
b.pending.put(takein.PendingKey(2), data)
}
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") {
t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys())
}
})
t.Run("T5 a take-in whose store fails once leaves no claim behind, and is taken in after", func(t *testing.T) {
claims := func(b *takeInBed) []string {
var out []string
for _, k := range b.keys() {
var p Pending
_ = json.Unmarshal(b.pending.vals[k], &p)
if p.Taking != "" {
out = append(out, k)
}
}
return out
}
once := func(match func(key string) bool) func(string) error {
done := false
return func(key string) error {
if !done && match(key) {
done = true
return errors.New("the store failed once")
}
return nil
}
}
for _, c := range []struct {
name string
fail func(b *takeInBed)
}{
// The second merge's claim fails: the first's, already made, is released.
{"a claim lost", func(b *takeInBed) {
b.pending.failWrite = once(func(k string) bool { return k == takein.PendingKey(3) })
}},
// The take-in itself fails on a module's write after both claims: both are released.
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
} {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
named := b.fp(t, delivery.IndexerModule)
b.ix.ownMu.Lock()
b.ix.ownFP = named
b.ix.ownMu.Unlock()
c.fail(b)
_, _, _ = b.ix.TakeInAtStart(ctx)
if got := claims(b); len(got) != 0 || len(b.keys()) != 2 {
t.Fatalf("%s: pending %v, claimed %v: want both pending and no claim left", c.name, b.keys(), got)
}
// The next take-in, not at a start, is not refused by a claim left behind: it takes both in.
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') {
t.Errorf("%s: after: pending %v, avahi at %s: want all taken in", c.name, b.keys(), b.commit(t, "avahi"))
}
}
})
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
entries, err := ReadPending(ctx, b.pending, nil)
if err != nil {
t.Fatal(err)
}
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
x := b.ix.dry().ix
for _, node := range []string{"novox", "ace"} {
key, _ := delivery.ToolchainKey(node, "go")
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false)
}
ready, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
next, _, err := x.computeOver(ctx, ready)
if err != nil || next == "" || next == b.said[0].WaitsFor {
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
}
b.ix.ownMu.Lock()
b.ix.ownFP = next
b.ix.ownMu.Unlock()
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.27.2")
if len(b.keys()) != 1 {
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
}
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.27.2")
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
var answer takein.PendingAnswer
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
if tool.Name == PendingTool {
out, err := tool.Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
answer = out.(takein.PendingAnswer)
}
}
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) ||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
t.Fatalf("indexer_pending = %+v", answer)
}
b.said = nil
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored {
t.Fatalf("drop: %+v", r)
}
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
}
b.said = nil
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
t.Fatalf("hand take-in: %+v", r)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
}
})
t.Run("1585 a merge over the value cap is held by reference, a new indexer with an empty memory takes it in through the forge bridge, and a covered pull request leaves its candidate", func(t *testing.T) {
big := func(f forgemsg.Forge) forgemsg.Forge {
f.Words = strings.Repeat("x", 300*1024)
f.Branch = "feat/g"
return f
}
b := newTakeInBed(t)
avahi0 := b.fp(t, "avahi")
// The forge bridge is not asked at all while the merges are in memory: a 300 KiB merge, over the bucket's cap whole,
// is held by reference and a lone one taken in.
bridgeDown := func(context.Context, string, string) ([]byte, error) {
return nil, errors.New("the bridge does not answer")
}
answers := b.ix.Message
b.ix.Message = bridgeDown
b.arrive(t, 1, big(b.merge(20, sha('b'), "tree-avahi-2", "")))
if b.fp(t, "avahi") == avahi0 || b.commit(t, "avahi") != sha('b') || len(b.keys()) != 0 || len(b.ix.held) != 0 {
t.Fatalf("a big merge alone: avahi at %s, pending %v, in memory %d: want it taken in and forgotten", b.commit(t, "avahi"), b.keys(), len(b.ix.held))
}
// Two pull requests of one group are open; both then merge (the indexer changed: held for the new build).
for n, head := range map[int]string{21: sha('1'), 22: sha('2')} {
if err := b.ix.Apply(ctx, pull(catalogueRepo, n, "feat/g", head, "modules/avahi")); err != nil {
t.Fatal(err)
}
}
key, _ := delivery.CandidatesKey("feat/g")
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
t.Fatal("the group has no candidate")
}
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
}
named := b.fp(t, delivery.IndexerModule)
// The new indexer starts with nothing in memory and the bridge down: the newest merge is said lost with the bridge's
// words (event, state), the older one is covered and not lost, and nothing is taken in.
next := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf,
GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }}
next.toolchains = b.ix.toolchains
next.FixOwnBuild(ctx)
b.said = nil
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
waits := PendingWaits(mustReadPending(t, b))
if len(b.said) != 1 || !slices.Equal(b.said[0].Lost, []string{takein.PendingKey(3)}) || len(waits) != 1 || !slices.Equal(waits[0].Lost, []string{takein.PendingKey(3)}) || b.commit(t, "avahi") != sha('b') {
t.Fatalf("lost: said %+v, waits %+v, avahi at %s: want m.3 lost on the event and in indexer_pending, nothing taken in", b.said, waits, b.commit(t, "avahi"))
}
if v, _, _, _ := b.pending.Get(ctx, takein.PendingKey(3)); !strings.Contains(string(v), "the bridge does not answer") {
t.Errorf("m.3 = %s: want the bridge's words", v)
}
if err := next.PendingAgain(ctx); err != nil || len(b.said) != 1 {
t.Errorf("lost again: err %v, said %v: want it said once", err, b.events())
}
if r := next.TakeInPendingByHand(ctx, takein.PendingKey(2), "lead@shanks", "try"); r.Stored || b.commit(t, "avahi") != sha('b') {
t.Errorf("hand take-in of a lost merge: %+v, avahi at %s: want nothing taken in", r, b.commit(t, "avahi"))
}
// The bridge answers: the newest merge is read through it, the older one is covered, and the revision is taken in.
b.publish(t, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) // the answer is the message as published, small here
next.Message = answers
b.bridge[catalogueRepo+"@"+sha('d')], _ = forgemsg.EncodeForge(big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
b.logs.Reset()
if err := next.PendingAgain(ctx); err != nil {
t.Fatal(err)
}
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || !strings.Contains(b.logs.String(), "is covered by") {
t.Fatalf("taken in: pending %v, avahi at %s, journal %q: want both merges taken in, the older covered", b.keys(), b.commit(t, "avahi"), b.logs.String())
}
if _, _, found, _ := b.ix.Store.Get(ctx, key); found {
t.Errorf("the group's candidate stays after both its pull requests merged (the covered one is not taken out)")
}
// A hand take-in of a merge by its commit asks the bridge and runs the message through Arrive.
b.ix.Message = answers
b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+delivery.IndexerModule])))
if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('e'), 9, "lead@shanks", "held by the cap"); !r.Stored || b.commit(t, "avahi") != sha('e') {
t.Fatalf("merge take-in: %+v, avahi at %s: want it taken in", r, b.commit(t, "avahi"))
}
if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('9'), 10, "lead@shanks", "no such"); r.Stored || !strings.Contains(r.Answer, "gate holds no merge") {
t.Errorf("merge take-in of a merge the bridge does not know: %+v", r)
}
// An old value that holds the message whole reads as before.
old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+delivery.IndexerModule])
rawOld, _ := forgemsg.EncodeForge(old)
data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, delivery.IndexerModule)})
if err := b.pending.Create(ctx, takein.PendingKey(11), data); err != nil {
t.Fatal(err)
}
if err := b.ix.PendingAgain(ctx); err != nil || b.commit(t, "avahi") != sha('f') {
t.Errorf("an old value with the message whole: err %v, avahi at %s: want it taken in", err, b.commit(t, "avahi"))
}
})
t.Run("the dry run names every field of the indexer", func(t *testing.T) {
// dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails.
named := map[string]bool{}
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections",
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery", "OwnBuildState"} {
named[f] = true
}
ty := reflect.TypeOf(Indexer{})
for i := range ty.NumField() {
if f := ty.Field(i); f.IsExported() && !named[f.Name] {
t.Errorf("Indexer.%s is not named by dry", f.Name)
}
}
})
}
func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry {
t.Helper()
entries, err := ReadPending(context.Background(), b.pending, nil)
if err != nil {
t.Fatal(err)
}
return entries
}
// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now.
func judgeSeatsHave(seat, verb string) bool {
s, _ := catalogue.SeatNamed(seat)
return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb })
}
// readPending is the bed's pending state as indexer_pending reads it.
func readPending(t *testing.T, b *takeInBed) []PendingEntry {
t.Helper()
entries, err := ReadPending(context.Background(), b.pending, nil)
if err != nil {
t.Fatal(err)
}
return entries
}
@@ -0,0 +1,205 @@
package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
stdio "git.novox.be/novox/mesh-sdk/go"
)
// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the
// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in
// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer.
func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
hand := func(drop bool) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}
}
merge := func(args map[string]any) (any, error) {
seq, _ := args["sequence"].(float64)
key, value, err := NewMergeAsk(str(args, "repository"), str(args, "merge"), uint64(seq), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Result, nil
}
return []stdio.Tool{
{Name: PendingTool,
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " +
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " +
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " +
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
var unread []string
entries, err := ReadPending(context.Background(), pending, func(format string, args ...any) {
unread = append(unread, fmt.Sprintf(format, args...))
})
if err != nil {
return nil, err
}
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
if ownBuild != nil {
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
var o OwnBuild
if json.Unmarshal(v, &o) == nil {
answer.OwnBuildSince = o.Since
if o.Known {
answer.OwnBuild = o.FP
} else {
answer.OwnBuildUnknown = o.Why
}
}
}
}
return answer, nil
}},
{Name: TakeInPendingTool,
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " +
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " +
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
Run: hand(false)},
{Name: DropPendingTool,
Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " +
"the pending state and nothing of it is taken in; what waited with it is worked out again. A hand act: who and why are " +
"said in the journal and on the take-in.taken event (dropped).",
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
"key": map[string]any{"type": "string", "description": "the pending merge's key, m.<sequence>, as indexer_pending names its merges' wait"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: hand(true)},
{Name: TakeInMergeTool,
Description: "Take one merge in by hand (ADR 0633 §5, issue 1585): the indexer asks the forge bridge for the merge's merged message " +
"(bridge_merged_message) and runs it through the take-in as a delivered merged message, so it is held for the " +
"indexer its revision names or taken in. For a merge the take-in did not take in (indexer_heard names its sequence " +
"and why). A hand act: who and why are said in the indexer's journal. The bridge's own words answer when it cannot " +
"build the message (the gate holds no such merge, the forge does not answer).",
Input: map[string]any{"type": "object", "required": []string{"repository", "merge", "sequence", "why"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository, owner/name"},
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"},
"sequence": map[string]any{"type": "integer", "description": "the merge's sequence on the forge bridge's stream, as indexer_heard names it"},
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
Run: merge},
}
}
// TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the
// candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint),
// and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it.
func TryCandidateTools(candidates, recovery Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
return []stdio.Tool{
{Name: TryCandidatesTool,
Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " +
"per try its heads hash, each member's repository and commit, the modules the try touches and, per module, its manifest digest, its " +
"fingerprint or why it has none yet (a merge preview not yet written). The checker's manifest validator and the node-engines' " +
"compose-candidate read this candidate. Read-only; an empty answer means no whole try waits or runs.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
ctx := context.Background()
keys, err := candidates.Keys(ctx)
if err != nil {
return nil, err
}
var out []delivery.Candidate
for _, k := range keys {
if !IsTryGroup(k) {
continue
}
v, _, found, err := candidates.Get(ctx, k)
if err != nil || !found {
continue
}
if c, err := delivery.DecodeCandidate(v); err == nil {
out = append(out, c)
}
}
// How the recovery of the open tries went: loud when it is not done (since when, why).
said := map[string]any{"done": false, "why": "no recovery has been written yet"}
if v, _, found, err := recovery.Get(ctx, TryRecoveryKey); err == nil && found {
var r TryRecovery
if json.Unmarshal(v, &r) == nil {
said = map[string]any{"done": r.Done}
if !r.Done {
said["why"], said["since"] = r.Why, r.Since
said["says"] = "recovery not done, since " + r.Since.Format(time.RFC3339) + ": " + r.Why
}
}
}
return map[string]any{"tries": out, "recovery": said}, nil
}},
{Name: RecoverTriesTool,
Description: "Run the recovery of the open tries now: the indexer lists them through the checker's checker_tries, writes the candidate of each " +
"whole try that waits or runs and deletes the candidate of a try no longer listed. It runs by itself at the indexer's start; when it failed " +
"(indexer_try_candidates says recovery not done, since when and why) this runs it again. A hand act: who and why are said in the journal.",
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is run now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
why, _ := args["why"].(string)
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("say why: a hand act is said in the journal")
}
data, err := json.Marshal(TakeInAsk{V: 1, RecoverTries: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("tries.recover."+now().UTC().Format("20060102T150405.000000000"), data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: DropTryCandidateTool,
Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " +
"is gone, or to make the indexer write it again from the previews at the try's next change. A hand act: who and why are said in the journal.",
Input: map[string]any{"type": "object", "required": []string{"try", "why"}, "properties": map[string]any{
"try": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
id, _ := args["try"].(string)
why, _ := args["why"].(string)
id = strings.TrimSpace(id)
if !IsTryGroup(id) || strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("name the try (try-<digits>) and say why")
}
data, err := json.Marshal(TakeInAsk{V: 1, TryCandidate: id, Drop: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("try-candidate.drop."+id, data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
}
}
@@ -72,7 +72,10 @@ func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, d
if !ix.PutPreview(key, value, deleted) {
return nil
}
return ix.RecomputeCandidates(ctx)
if err := ix.RecomputeCandidates(ctx); err != nil {
return err
}
return ix.WriteTryCandidates(ctx) // a try that waited for this preview gets its candidate
}
// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
@@ -3,10 +3,17 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
@@ -38,7 +45,7 @@ func avahiOnTools(t *testing.T) string {
}
// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest.
func avahiPull(t *testing.T, head string) delivery.Forge {
func avahiPull(t *testing.T, head string) forgemsg.Forge {
t.Helper()
f := manifestPull(t, 301, head, avahiOnTools(t))
f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)}
@@ -181,6 +188,256 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview {
t.Errorf("a new head: %+v, want no preview", got)
}
// Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try events and
// the previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates,
// and deletes it when the try ends. Its manifests are judged with the seat definitions its members carry, in one locked
// pass, whether the contracts module's manifest changed or only its seat.json did (the wait is bounded: a lock held twice
// would show as a failure here, not as a stuck run).
for _, row := range []struct {
name string
unchanged bool // the contracts module's module.json is the trunk's: only its definitions changed
}{
{name: "a changed manifest that carries definitions"},
{name: "an unchanged manifest that carries definitions", unchanged: true},
} {
t.Run("a group try's candidate: "+row.name, func(t *testing.T) {
ix, mods, cands := previewBed(t, true)
ctx := context.Background()
ix.Verdicts = newMem()
var changed map[string]any
if err := json.Unmarshal([]byte(catalogueManifest(t, "avahi")), &changed); err != nil {
t.Fatal(err)
}
changed["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go-seats", "kind": "package", "language": "go", "paths": []any{"go"}, "generated-from": "defs"}}}
raw, _ := json.Marshal(changed)
pv := avahiPreview(t, trunkNow, h1)
pm := delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest(raw), Locks: map[string]string{"go.sum": avahiGoSum}}
if row.unchanged {
m, found := moduleOf(t, mods, "avahi")
_ = found
pm.Manifest = m.Manifest
key, _ := delivery.ManifestsKey(m.Manifest)
data, _ := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(raw)})
if _, _, _, err := ix.Manifests.Get(ctx, key); err != nil {
t.Fatal(err)
}
_ = ix.Manifests.(*memStore).Update(ctx, key, data, ix.Manifests.(*memStore).revs[key])
}
pv.Modules["modules/avahi"] = pm
ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) {
return RepoRead{Manifests: map[string]string{dir: string(raw)}}, nil
}
listed := 0
ix.Files = func(_ context.Context, _, commit string, dirs []string) (map[string]map[string]string, error) {
listed++
if commit != h1 || !slices.Equal(dirs, []string{"modules/avahi/defs/seats"}) {
t.Errorf("listed %v at %s, want the member's head", dirs, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
}
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case "modules/avahi/defs/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
case "modules/avahi/defs/parts/reads.json":
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"try-verb"}]}`), nil
}
return nil, nil
}
started := `{"id":"try-7"}`
_ = started
members := []TryMember{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}
held := func() []string { k, _ := cands.Keys(ctx); return k }
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 0 {
t.Fatalf("a try before its preview: %v, candidates %v", err, held())
}
done := make(chan struct{})
go func() { defer close(done); applyPreview(t, ix, pv) }()
select {
case <-done:
case <-time.After(10 * time.Second):
panic("writing the try's candidate hangs: the seat lock is held twice (a locked pass cannot be left by t.Fatal: the lock stays held)")
}
if listed == 0 {
t.Error("the member's definitions were not read")
}
key, _ := delivery.CandidatesKey("try-7")
rawc, _, found, _ := cands.Get(ctx, key)
if !found {
t.Fatalf("no candidate for the try after its preview: %v", held())
}
c, err := delivery.DecodeCandidate(rawc)
pull := delivery.CandidatePull{}
if len(c.Pulls) == 1 {
pull = c.Pulls[0]
}
if row.unchanged {
if err != nil || c.Group != "try-7" || len(pull.Modules) != 0 {
t.Fatalf("candidate = %+v, %v, want the try's group with no module touched", c, err)
}
} else if err != nil || c.Group != "try-7" || pull.Number != 0 || pull.Head.Head != h1 || pull.Fps["avahi"].Manifest == "" { // no fingerprint asserted: this mesh-sdk pin does not read generated-from yet
t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's manifest", c, err)
}
if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 {
t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err)
}
// The tools: one reads the candidates held for tries, one drops by an ask the indexer answers.
asks := func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
a.Result = &r
return a, err
}
recovery := newMem()
ix.TryRecovery = recovery
tools := TryCandidateTools(cands, recovery, asks, time.Now)
if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 {
t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err)
}
if _, err := tools[2].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil {
t.Error("a branch was dropped as a try")
}
if out, err := tools[2].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
t.Errorf("drop = %v, %v, candidates %v", out, err, held())
}
// The start recovered from the checker's tool: an open whole try is written again, one the checker no longer
// lists is deleted, and a try narrowed to a level is none.
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
{ID: "try-8", Whole: false, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h2, Trunk: trunkNow}}}}, nil
}
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 1 {
t.Fatalf("recovered: %v, candidates %v, want try-7's alone", err, held())
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 {
t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held())
}
// A recovery whose call fails is loud and deletes nothing: the candidate stays, indexer_try_candidates says recovery not
// done since when and why, and the next try event runs the recovery again, which clears it; the tool runs it by hand.
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
t.Fatalf("started: %v, candidates %v", err, held())
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return nil, errors.New("checker_tries does not answer")
}
if err := ix.RecoverTries(ctx); err == nil || len(held()) != 1 {
t.Fatalf("a failed call: %v, candidates %v, want the error and the candidate kept", err, held())
}
if out, err := tools[0].Run(map[string]any{}); err != nil || !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "recovery not done, since") ||
!strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "checker_tries does not answer") {
t.Errorf("%s = %v, %v, want recovery not done with since and why", tools[0].Name, out, err)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if err := ix.StartTry(ctx, "try-11", members); err != nil || len(held()) != 1 || held()[0] != "try-11" {
t.Fatalf("a try event after a failed recovery: %v, candidates %v, want the recovery run (try-7 not listed, gone) and try-11 written", err, held())
}
if out, _ := tools[0].Run(map[string]any{}); out.(map[string]any)["recovery"].(map[string]any)["done"] != true {
t.Errorf("recovery still said not done after a success: %v", out)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, errors.New("down") }
recoverAsk := func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
r := ix.RecoverTriesByHand(ctx, a.By, a.Why)
a.Result = &r
return a, err
}
byHand := TryCandidateTools(cands, recovery, recoverAsk, time.Now)
if out, err := byHand[1].Run(map[string]any{"why": "the checker was replaced"}); err != nil || out.(*TakenIn).Stored || !strings.Contains(out.(*TakenIn).Answer, "recovery not done") {
t.Errorf("%s with the checker down = %v, %v", byHand[1].Name, out, err)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if out, err := byHand[1].Run(map[string]any{"why": "the checker is back"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
t.Errorf("%s = %v, %v, candidates %v, want it recovered (try-11 not listed)", byHand[1].Name, out, err, held())
}
if err := ix.StartTry(ctx, "try-7", members); err != nil {
t.Fatal(err)
}
// ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after
// it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a
// try that ended during the call, listed as open, leaves none once its end is heard.
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
{ID: "try-9", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}}, nil
}
if err := ix.RecoverTries(ctx); err != nil {
t.Fatal(err)
}
for _, step := range []struct {
what string
do func() error
want int
}{
{"the start heard after the call, of a try the call listed", func() error { return ix.StartTry(ctx, "try-7", members) }, 2},
{"a start after the list", func() error { return ix.StartTry(ctx, "try-10", members) }, 3},
{"the end heard of a try ended during the call", func() error { return ix.EndTry(ctx, "try-9") }, 2},
} {
if err := step.do(); err != nil || len(held()) != step.want {
t.Fatalf("%s: %v, candidates %v, want %d", step.what, err, held(), step.want)
}
}
for _, id := range []string{"try-10"} {
if err := ix.EndTry(ctx, id); err != nil {
t.Fatal(err)
}
}
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
t.Fatalf("started again: %v, candidates %v", err, held())
}
if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 {
t.Errorf("an ended try: %v, candidates %v, want none", err, held())
}
// The hand recovery never overlaps a try event: a start heard while the call to the checker is out writes its candidate
// before or after the sweep, never between the call and the sweep (which would delete it, the try not being listed).
startDone := make(chan error, 1)
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
go func() { startDone <- ix.StartTry(ctx, "try-12", members) }()
for i := 0; i < 2000 && len(held()) == 0; i++ {
runtime.Gosched() // lets the start write if nothing holds it back
}
return nil, nil
}
if r := ix.RecoverTriesByHand(ctx, "lead", "a test"); !r.Stored {
t.Fatalf("the hand recovery = %+v", r)
}
if err := <-startDone; err != nil || len(held()) != 1 || held()[0] != "try-12" {
t.Errorf("a start heard during the hand recovery: %v, candidates %v, want try-12 kept", err, held())
}
})
}
// Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes;
// any other failure (the forge does not answer) is asked again at every pass and said once per head.
for _, row := range []struct {
name string
err error
askedAfterTwo int // the same head read twice
askedAfterNew int // then another head
linesAfterNew int
}{
{"the directory is not in the tree", fmt.Errorf("listing: %w", ErrNotInTree), 1, 2, 2},
{"the forge does not answer", errors.New("the forge does not answer"), 2, 3, 2},
} {
t.Run("a failing head: "+row.name, func(t *testing.T) {
ix, _, _ := previewBed(t, false)
var logs strings.Builder
ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") }
asked := 0
ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) {
asked++
return nil, row.err
}
held := map[string]delivery.Module{"x": {Inputs2: &delivery.KeptInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}}
for pass, head := range []string{h1, h1, h2} {
ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held)
if pass == 1 && (asked != row.askedAfterTwo || strings.Count(logs.String(), "its packages are not read") != 1) {
t.Fatalf("after two passes over %s: asked %d (want %d), journal %q: want one line", head, asked, row.askedAfterTwo, logs.String())
}
}
if asked != row.askedAfterNew || strings.Count(logs.String(), "its packages are not read") != row.linesAfterNew {
t.Errorf("asked %d (want %d), journal %q (want %d lines)", asked, row.askedAfterNew, logs.String(), row.linesAfterNew)
}
})
}
}
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
@@ -7,6 +7,7 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -125,7 +126,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
return t, nil
}
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
Trees: map[string]string{dir: read.Trees[dir]},
Manifests: map[string]string{dir: raw}}
if l := read.Locks[dir]; len(l) > 0 {
@@ -396,7 +397,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: out.Repository, Merge: head}
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head}
if _, err := ix.ApplyMessage(ctx, f); err != nil {
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
return out
@@ -0,0 +1,154 @@
package index
// The seat definitions a revision carries (novox/hq ADR 0633 §1, ADR 0635 §3, ADR 0636 §4): when a take-in works out which
// indexer a revision names, the manifests it writes are judged against the seats of that revision, never against the
// list this program was built with, since the indexer the revision names may serve or invoke what the revision's
// definitions add (a verb of a seat, an event). They are read from the merges' trees: each contracts module a merge's
// manifests name (a package artifact with `generated-from`) holds `seats/<seat>/seat.json` and the `parts/` they
// include, read through the forge's listing and get-file tools. A delivery group's merges are one revision, so a
// group's members' definitions are read together.
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"path"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
)
// definitionsDir is the directory of a contracts module's definitions that the module.json at dir names, from the first
// package artifact naming `generated-from`: ok is false for a module that is no contracts module. A manifest that does
// not decode is none here: the judge says why it is refused.
func definitionsDir(dir, manifest string) (string, bool) {
var m struct {
Build struct {
Artifacts []struct {
GeneratedFrom string `json:"generated-from"`
} `json:"artifacts"`
} `json:"build"`
}
if json.Unmarshal([]byte(manifest), &m) != nil {
return "", false
}
for _, a := range m.Build.Artifacts {
if a.GeneratedFrom == "" {
continue
}
base := dir
if base == "." || base == "/" {
base = ""
}
return path.Clean(path.Join(base, a.GeneratedFrom)), true
}
return "", false
}
// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge
// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error
// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it
// would say it unparsed.
func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) {
if ix.ReadFile == nil || ix.Files == nil {
return nil, nil
}
var defs []judge.Definition
for _, e := range ready {
if e.Covered || e.Forge.Merge == "" {
continue
}
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
defsDir, ok := definitionsDir(dir, e.Forge.Manifests[dir])
if !ok {
continue
}
got, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, defsDir)
if err != nil {
return nil, err
}
defs = append(defs, got...)
}
}
seats, err := judge.Seats(defs)
if err != nil {
return nil, err
}
return seats, nil
}
// readDefinitions reads the definitions under a contracts module's definitions directory at a commit, resolved with their
// parts.
func (ix *Indexer) readDefinitions(ctx context.Context, repository, commit, defsDir string) ([]judge.Definition, error) {
file := func(p string) ([]byte, error) {
if defsDir != "." && defsDir != "" {
p = defsDir + "/" + p
}
raw, err := ix.ReadFile(ctx, repository, commit, p)
if err != nil {
return nil, fmt.Errorf("reading %s of %s at %.12s: %w", p, repository, commit, err)
}
return raw, nil
}
dir := path.Join(defsDir, "seats")
if defsDir == "." || defsDir == "" {
dir = "seats"
}
listed, err := ix.Files(ctx, repository, commit, []string{dir})
if err != nil {
return nil, fmt.Errorf("listing %s of %s at %.12s: %w", dir, repository, commit, err)
}
parts := map[string]judge.Part{}
var out []judge.Definition
for _, p := range slices.Sorted(maps.Keys(listed[dir])) {
seat, ok := strings.CutSuffix(p, "/seat.json")
if !ok || seat == "" || strings.Contains(seat, "/") {
continue
}
raw, err := file("seats/" + seat + "/seat.json")
if err != nil {
return nil, err
}
d, err := judge.ParseDefinition(raw)
if err == nil && d.Seat != seat {
err = fmt.Errorf("it names the seat %q, not %q", d.Seat, seat)
}
for _, name := range d.Includes {
if err != nil {
break
}
if _, have := parts[name]; have {
continue
}
praw, perr := file("parts/" + name + ".json")
if perr != nil {
return nil, perr
}
var part judge.Part
if praw == nil {
perr = errors.New("the commit holds no such part")
} else {
part, perr = judge.ParsePart(praw)
}
if perr != nil {
err = fmt.Errorf("its part %s: %w", name, perr)
break
}
parts[name] = part
}
if err == nil {
d, err = d.Resolve(parts)
}
if err != nil {
ix.logf("take-in: the seat definition seats/%s of %s at %.12s does not parse, so it is left out: %v", seat, repository, commit, err)
continue
}
out = append(out, d)
}
return out, nil
}
@@ -51,6 +51,17 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
}
func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first)
// A source revision older than the one held is left (merged here from a test of its own, for the test budget).
t.Run("an older source revision is left", func(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
ctx := context.Background()
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
_ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge))
_ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge))
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
t.Errorf("module a Src = %d, want 9", m.Src)
}
})
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
ctx := context.Background()
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil {
@@ -93,14 +104,3 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344
t.Errorf("module a = %+v, want the new merge with Src 0", m)
}
}
func TestASourceRevisionOlderThanTheOneHeldIsLeft(t *testing.T) {
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
ctx := context.Background()
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
_ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge))
_ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge))
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
t.Errorf("module a Src = %d, want 9", m.Src)
}
}
@@ -4,6 +4,7 @@ import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"time"
@@ -33,11 +34,24 @@ type TakeInAsk struct {
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
Judge bool `json:"judge,omitempty"`
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
Digest string `json:"digest,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
Digest string `json:"digest,omitempty"`
// Pending asks the wait under this take-in key to be taken in by hand (indexer_take_in_pending), or with Drop the
// one pending merge under it to be dropped (indexer_drop_pending), instead of a module (pending.go, ADR 0633 §5).
Pending string `json:"pending,omitempty"`
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
Merge string `json:"merge,omitempty"`
MergeRepository string `json:"merge-repository,omitempty"`
Sequence uint64 `json:"sequence,omitempty"`
// TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go.
TryCandidate string `json:"try-candidate,omitempty"`
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
RecoverTries bool `json:"recover-tries,omitempty"`
Drop bool `json:"drop,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// Done says the ask is answered.
@@ -52,8 +66,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
if a.V != 1 {
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
}
if a.Module == "" && a.Repository == "" {
return a, errors.New("a takeins value naming no module or repository")
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries {
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
}
return a, nil
}
@@ -82,3 +96,49 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by
data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "repository." + key, data, err
}
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
const TakeInMergeTool = "indexer_take_in_merge"
var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`)
// NewMergeAsk is the value indexer_take_in_merge writes: its key in takeins, `merge.<commit>`, and the value. sequence is
// the merge's sequence on the forge bridge's stream, as indexer_heard names it: the pending merge is keyed by it, which
// keeps the merge order.
func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at time.Time) (string, []byte, error) {
repository, merge = strings.TrimSpace(repository), strings.TrimSpace(merge)
switch {
case !strings.Contains(repository, "/"):
return "", nil, errors.New("name the repository as owner/name (repository)")
case !sha40.MatchString(merge):
return "", nil, errors.New("name the merge commit, 40 hex digits (merge)")
case sequence == 0:
return "", nil, errors.New("name the merge's sequence on the forge bridge's stream (sequence), as indexer_heard names it")
case strings.TrimSpace(why) == "":
return "", nil, errors.New("say why (why): a hand act is said in the journal")
}
data, err := json.Marshal(TakeInAsk{V: 1, Merge: merge, MergeRepository: repository, Sequence: sequence, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "merge." + merge, data, err
}
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
// drop one pending merge's.
func NewPendingAsk(key, why, by string, drop bool, at time.Time) (string, []byte, error) {
key = strings.TrimSpace(key)
if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) {
if drop {
return "", nil, errors.New("name the pending merge to drop by its key, m.<sequence>, as indexer_pending names it")
}
return "", nil, errors.New("name the wait to take in by its key, m.<sequence> or g.<group>, as indexer_pending names it")
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event")
}
act := "take-in"
if drop {
act = "drop"
}
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, Drop: drop, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return "pending." + act + "." + key, data, err
}
@@ -8,6 +8,7 @@ import (
"slices"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -19,13 +20,13 @@ import (
// say keeps the key as it is and says so.
// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one.
func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error {
func (ix *Indexer) writeTaken(ctx context.Context, f forgemsg.Forge) error {
return ix.writeTakenKnown(ctx, f, "")
}
// writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a
// key naming that merge is written over without asking the forge again.
func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOlder string) error {
func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOlder string) error {
if ix.Taken == nil || f.Merge == "" {
return nil
}
@@ -101,7 +102,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error {
}
}
for _, repository := range slices.Sorted(maps.Keys(pending)) {
err := ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: pending[repository]})
err := ix.writeTaken(ctx, forgemsg.Forge{Repository: repository, Merge: pending[repository]})
if err == nil {
ix.mu.Lock()
if ix.untaken[repository] == pending[repository] {
@@ -78,12 +78,22 @@ func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool)
}
// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain
// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing.
// changed, writes the modules again that it moves and evaluates pending again (ToolchainReported). The same version and
// recipe seen again writes nothing.
func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error {
if !ix.PutToolchain(module, key, value, deleted) {
return nil
}
return ix.Recompute(ctx)
return ix.ToolchainReported(ctx)
}
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute), then pending evaluated
// again (PendingAgain), since the indexer build a pending revision names is a fingerprint the toolchain is part of.
func (ix *Indexer) ToolchainReported(ctx context.Context) error {
if err := ix.Recompute(ctx); err != nil {
return err
}
return ix.PendingAgain(ctx)
}
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
@@ -10,6 +10,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -91,7 +92,7 @@ func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
}
// newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied.
func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, delivery.Forge) {
func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, forgemsg.Forge) {
t.Helper()
var recs [][2]any
trees, manifests := map[string]string{}, map[string]string{}
@@ -468,7 +469,7 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
tools, _ := moduleOf(t, mods, "mesh-tools")
sdkMerge := func(commit, tree string) delivery.Module {
t.Helper()
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1"}`}}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
@@ -0,0 +1,439 @@
package index
// **A group try's candidate** (issue 1921; novox/hq ADR 0593 §7): a try of several repositories at exact commits has no pull
// request, so the forge bridge sends the indexer no message for it; yet the node-engine's compose-candidate and the checker's
// manifest validator read the indexer's candidate of a group. The indexer therefore watches the checker's state tries (the
// checker keeps it; this module reads it) and, for a try that is not narrowed to a level or to some tests, writes
// `candidates.<try id>`: one pull per member (number 0), its modules and manifests taken from the merge preview the try's
// fetch wrote, its manifests kept and judged with the seat definitions the members carry, as a take-in judges a revision's.
// The fingerprints come with the previews (RecomputeCandidates). The candidate is deleted when the try ends. `try-<digits>`
// is the reserved group name of a try: the bridge's open message never takes it out, and the checker never checks it.
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// TryCandidateTools: the tools that read, drop and recover a try's candidate.
const (
TryCandidatesTool = "indexer_try_candidates"
DropTryCandidateTool = "indexer_drop_try_candidate"
RecoverTriesTool = "indexer_recover_tries"
// TryRecoveryState is the indexer's state of how the last recovery of the open tries went: the bucket
// mesh-delivery-indexer_try-recovery, one key (TryRecoveryKey).
TryRecoveryState = "try-recovery"
TryRecoveryBucket = delivery.IndexerModule + "_" + TryRecoveryState
TryRecoveryKey = "state"
)
// TryRecovery is the value of the try-recovery state: whether the candidates of the open tries were recovered at the
// indexer's start or by hand, and when it began to fail, why.
type TryRecovery struct {
V int `json:"v"`
Done bool `json:"done"`
Since time.Time `json:"since,omitzero"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
}
var tryGroupOK = regexp.MustCompile(`^try-[0-9]{1,20}$`)
// IsTryGroup says a candidates key is the group of a try.
func IsTryGroup(key string) bool { return tryGroupOK.MatchString(key) }
// TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto.
type TryMember struct{ Repository, Head, Trunk string }
// StartTry takes a whole try in, as the checker said it started (tryevent.Started): its members at their commits are kept, and
// its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596).
func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
ix.retryRecovery(ctx)
return ix.startTry(ctx, id, members)
}
// retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds.
func (ix *Indexer) retryRecovery(ctx context.Context) {
if ix.recoveryFailed.Load() && ix.OpenTries != nil {
_ = ix.recoverNow(ctx)
}
}
func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) error {
if !IsTryGroup(id) || len(members) == 0 {
return fmt.Errorf("%q with %d members is no try to start", id, len(members))
}
ix.mu.Lock()
if ix.tries == nil {
ix.tries = map[string][]TryMember{}
}
ix.tries[id] = slices.Clone(members)
ix.mu.Unlock()
return ix.writeTryCandidate(ctx, id)
}
// EndTry takes a try out, as the checker said it ended (tryevent.Ended), and deletes its candidate.
func (ix *Indexer) EndTry(ctx context.Context, id string) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
ix.retryRecovery(ctx)
return ix.endTry(ctx, id)
}
func (ix *Indexer) endTry(ctx context.Context, id string) error {
ix.mu.Lock()
delete(ix.tries, id)
ix.mu.Unlock()
if !IsTryGroup(id) {
return nil
}
return ix.dropTryCandidate(ctx, id)
}
// RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait
// or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them
// (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask.
//
// A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds:
// the events are still heard after it, and indexer_recover_tries runs it again by hand.
//
// A recovery and a try event never overlap (tryMu): a sweep that deleted the candidate a start event wrote after the call to the
// checker would leave a running try with none.
func (ix *Indexer) RecoverTries(ctx context.Context) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
return ix.recoverNow(ctx)
}
func (ix *Indexer) recoverNow(ctx context.Context) error {
if ix.OpenTries == nil || ix.Store == nil {
return nil
}
err := ix.recoverTries(ctx)
ix.recoveryFailed.Store(err != nil)
ix.noteRecovery(ctx, err)
return err
}
// noteRecovery writes how a recovery went: failing keeps the time it began to fail, and the journal says it each time.
func (ix *Indexer) noteRecovery(ctx context.Context, err error) {
now := ix.now()
r := TryRecovery{V: 1, Done: err == nil, At: now.UTC()}
if err != nil {
r.Why, r.Since = err.Error(), now.UTC()
ix.logf("RECOVERY OF THE TRIES NOT DONE: %v (the try candidates may be stale or missing until indexer_recover_tries or the next start succeeds)", err)
}
if ix.TryRecovery == nil {
return
}
for range writeTries {
cur, rev, found, gerr := ix.TryRecovery.Get(ctx, TryRecoveryKey)
if gerr != nil {
ix.logf("the try-recovery state cannot be read: %v", gerr)
return
}
if err != nil && found {
var was TryRecovery
if json.Unmarshal(cur, &was) == nil && !was.Done && !was.Since.IsZero() {
r.Since = was.Since
}
}
data, _ := json.Marshal(r)
var werr error
if found {
werr = ix.TryRecovery.Update(ctx, TryRecoveryKey, data, rev)
} else {
werr = ix.TryRecovery.Create(ctx, TryRecoveryKey, data)
}
if werr == nil {
return
}
if !errors.Is(werr, ErrConflict) {
ix.logf("the try-recovery state cannot be written: %v", werr)
return
}
}
}
// RecoverTriesByHand runs the recovery now (indexer_recover_tries), a hand act said in the journal.
func (ix *Indexer) RecoverTriesByHand(ctx context.Context, by, why string) TakenIn {
r := TakenIn{Module: "tries"}
ix.logf("recovery of the tries asked by %s (%s)", by, why)
if ix.OpenTries == nil {
r.Answer = "the indexer is wired to list no open try: nothing to recover from"
return r
}
if err := ix.RecoverTries(ctx); err != nil {
r.Answer = "recovery not done: " + err.Error()
return r
}
r.Stored, r.Answer = true, "the candidates of the open tries are recovered"
return r
}
func (ix *Indexer) recoverTries(ctx context.Context) error {
open, err := ix.OpenTries(ctx)
if err != nil {
return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err)
}
live := map[string]bool{}
for _, o := range open {
if !o.Whole || len(o.Members) == 0 {
continue
}
live[o.ID] = true
var ms []TryMember
for _, m := range o.Members {
ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk})
}
if err := ix.startTry(ctx, o.ID, ms); err != nil {
return err
}
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return err
}
for _, k := range keys {
if IsTryGroup(k) && !live[k] {
if err := ix.endTry(ctx, k); err != nil {
return err
}
}
}
return nil
}
// WriteTryCandidates writes the candidate of every live try that has none yet and whose previews are all there: after a
// preview was written, the try that waited for it.
func (ix *Indexer) WriteTryCandidates(ctx context.Context) error {
ix.mu.Lock()
ids := make([]string, 0, len(ix.tries))
for id := range ix.tries {
ids = append(ids, id)
}
ix.mu.Unlock()
slices.Sort(ids)
for _, id := range ids {
if err := ix.writeTryCandidate(ctx, id); err != nil {
return err
}
}
return nil
}
func (ix *Indexer) dropTryCandidate(ctx context.Context, id string) error {
if ix.Store == nil {
return nil
}
key, err := delivery.CandidatesKey(id)
if err != nil {
return err
}
return write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil })
}
// DropTryCandidate deletes a try's candidate by hand (indexer_drop_try_candidate); a live try gets it again at its next
// change.
func (ix *Indexer) DropTryCandidate(ctx context.Context, id, by, why string) TakenIn {
r := TakenIn{Module: id}
switch {
case !IsTryGroup(id):
r.Answer = id + " is no try's group, try-<digits>: nothing dropped"
default:
if err := ix.dropTryCandidate(ctx, id); err != nil {
r.Answer = "the candidate of " + id + " could not be dropped: " + err.Error()
} else {
r.Stored, r.Answer = true, "the candidate of "+id+" is dropped by "+by+" ("+why+")"
ix.logf("%s", r.Answer)
}
}
return r
}
// writeTryCandidate writes the candidate of a live try when it has none at its members' heads and every member's merge
// preview is held.
func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error {
if ix.Store == nil || ix.Read == nil {
return nil
}
ix.mu.Lock()
members := slices.Clone(ix.tries[id])
ix.mu.Unlock()
if len(members) == 0 {
return nil
}
key, err := delivery.CandidatesKey(id)
if err != nil {
return err
}
slices.SortFunc(members, func(a, b TryMember) int { return strings.Compare(a.Repository, b.Repository) })
heads := make([]delivery.Head, len(members))
for i, m := range members {
heads[i] = delivery.Head{Repository: m.Repository, Head: m.Head}
}
hash, err := delivery.HeadsHash(heads)
if err != nil {
return err
}
if value, _, found, err := ix.Store.Get(ctx, key); err == nil && found {
if c, err := delivery.DecodeCandidate(value); err == nil && c.HeadsHash == hash {
return nil // fingerprints follow the previews (RecomputeCandidates)
}
}
previews := make([]delivery.Preview, len(members))
for i, m := range members {
pk, err := delivery.PreviewKey(m.Repository, m.Head, m.Trunk)
if err != nil {
return fmt.Errorf("the try %s: %w", id, err)
}
ix.mu.Lock()
pv, ok := ix.previews[pk]
ix.mu.Unlock()
if !ok {
return nil // its fetch has not written the preview yet: this is called again when it does
}
previews[i] = pv
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
pulls := make([]delivery.CandidatePull, len(members))
var entries []PendingEntry
over := map[string]string{}
for i, m := range members {
pull := delivery.CandidatePull{Head: heads[i], Modules: []string{}}
// defs are every module directory's manifest of the member at its head, the unchanged ones too: a contracts module whose
// seat.json changed while its module.json did not carries definitions all the same (ADR 0636 §4).
defs := map[string]string{}
for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) {
pm := previews[i].Modules[dir]
name, unchanged := "", false
for n, h := range held {
if delivery.SameRepository(h.Repository, m.Repository) && h.Path == forgeDir(dir) {
name, unchanged = n, h.Manifest == pm.Manifest
break
}
}
if unchanged {
if raw := ix.keptManifest(ctx, pm.Manifest); raw != "" {
defs[dir] = raw
}
continue // the manifest the trunk holds: the try does not touch the module
}
rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir))
if err != nil {
return fmt.Errorf("the try %s: reading the manifest of %s at %.12s: %w", id, dir, m.Head, err)
}
raw := rr.Manifests[forgeDir(dir)]
if raw == "" {
ix.logf("the try %s: %s holds no manifest at %.12s, so no module of it is taken", id, dir, m.Head)
continue
}
module := manifestModule(raw)
if module == "" {
module = name
}
if module == "" {
module = path.Base(strings.Trim(dir, "/."))
}
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
return err
}
if digest != pm.Manifest {
ix.logf("the try %s: the manifest of %s read at %.12s is not the one its preview names, left", id, dir, m.Head)
continue
}
if pull.Fps == nil {
pull.Fps = map[string]delivery.CandidateModule{}
}
pull.Modules = append(pull.Modules, module)
pull.Fps[module] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
defs[dir] = raw
over[module] = digest
}
pulls[i] = pull
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: defs}})
}
// The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4), in one pass that takes the
// seats. Their verdicts are the verdicts state's, keyed by the manifest's digest, as a pull request's candidate's are
// (judgeDigests in ApplyMessage): nothing reads them but the engines' composition of that digest and the module
// value of a trunk module with the same digest.
seats, err := ix.revisionSeats(ctx, entries)
if err != nil {
return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err)
}
ix.judgeDigestsWith(ctx, seats, over, "the try "+id)
c := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: hash, Pulls: pulls}
c.FPSet = delivery.FingerprintSet(c.Pulls)
ix.withPreviews(ctx, &c)
if err := ix.refuseDisagreement(c); err != nil {
return err
}
data, err := delivery.EncodeCandidate(c)
if err != nil {
return err
}
return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil })
}
// keptManifest is the text of the manifest the indexer keeps under a digest, or empty.
func (ix *Indexer) keptManifest(ctx context.Context, digest string) string {
if ix.Manifests == nil || digest == "" {
return ""
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return ""
}
value, _, found, err := ix.Manifests.Get(ctx, key)
if err != nil || !found {
return ""
}
m, err := delivery.DecodeManifest(value)
if err != nil {
return ""
}
return m.JSON
}
func mapKeys[V any](m map[string]V) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
// forgeDir is a preview's directory as the forge reader spells it: the root as "".
func forgeDir(dir string) string {
if dir == "." || dir == "/" {
return ""
}
return strings.Trim(dir, "/")
}
// heldAt says the trunk holds a module of the repository in that directory.
func heldAt(held map[string]delivery.Module, repository, dir string) bool {
for _, h := range held {
if delivery.SameRepository(h.Repository, repository) && h.Path == forgeDir(dir) {
return true
}
}
return false
}

Some files were not shown because too many files have changed in this diff Show More