mesh/merge-gate pass: the change touches no module of the mesh's graph
mesh/repo-check pass: its merge-check.sh passed, each part in its toolchain: merge-check.sh (typescript), replays/merge-check.sh (go)
mesh/delivery delivered
mesh/delivery-group group replays/331-a-tool-check-on-the-live-link delivered: every member is delivered
hq issue 331: every declared tool check read no link to the bus is open while the node-engine's link was up. Proved: fails on 824911d^1, passes on 824911d.
266 lines
20 KiB
Go
266 lines
20 KiB
Go
// Package replays is the lab's replay of every core incident (novox/hq to-be 45 §9, M9): a scripted
|
|
// replay of what happened, asserting the rule's outcome, run before every merge of a core repository —
|
|
// and proved, once, to fail on the commit before its fix and to pass on the fix.
|
|
//
|
|
// **Where a replay lives.** A replay of one component's logic is a test in that component's repository,
|
|
// written only with what the component had before its fix, so it can be laid over the older commit
|
|
// (the controller's replays_test.go). A replay of what the mesh runs rather than what it wrote — the bus
|
|
// server's release, the resolver the catalogue configures, under both C libraries — lives here, where a
|
|
// container runtime and a bus of a given release are at hand. Every one is in Register, with the issue,
|
|
// the fix, and how the prover runs it at a commit.
|
|
//
|
|
// **A new core issue resolves with its replay added here, or with a stated reason none is possible**
|
|
// (hq 00-META/checks/cycle.py holds the issue to it: `replay:` names an entry here, or `replay-none:`
|
|
// says why).
|
|
package replays
|
|
|
|
// Kind is how a replay is run at a commit.
|
|
type Kind string
|
|
|
|
const (
|
|
// InRepository is a test in the fixed repository, laid over the commit and run there with go test.
|
|
InRepository Kind = "test"
|
|
// Bus is the bus replay here, run against a server of the release the catalogue pinned at the commit.
|
|
Bus Kind = "bus"
|
|
// Resolver is the resolver replay here, run with the catalogue's resolver configuration at the commit,
|
|
// asked by a program under musl and one under glibc.
|
|
Resolver Kind = "resolver"
|
|
// Gate is a test of the merge gate itself: before its fix there was no check to fail, so the commit
|
|
// before is held to fail by not having it.
|
|
Gate Kind = "gate"
|
|
// Liveness is the crash-loop replay here: a container that exits at start, raised on the runtime here,
|
|
// judged by the node-engine at the commit, and the controller at the commit judging its gate from what
|
|
// the engine said (novox/hq ADR 0240). Two repositories move together: Repository and With.
|
|
Liveness Kind = "liveness"
|
|
)
|
|
|
|
// Also is another repository a replay runs at its own commits beside Repository's: before its fix and on
|
|
// it, as Repository is.
|
|
type Also struct {
|
|
Repository string
|
|
Fix string
|
|
// Before is the commit the replay runs this repository at before the fix, when that is not Fix's
|
|
// first parent.
|
|
Before string
|
|
}
|
|
|
|
// Replay is one incident, replayed.
|
|
type Replay struct {
|
|
ID string
|
|
Issue int
|
|
// What is the incident in a line, and Asserts the rule's outcome the replay holds.
|
|
What, Asserts string
|
|
Kind Kind
|
|
// Repository is where the fix is, and Fix the fix's commit there: the replay must fail on Fix's first
|
|
// parent and pass on Fix.
|
|
Repository string
|
|
Fix string
|
|
// Before is the commit the replay must fail on, when that is not Fix's first parent: a fix still on its
|
|
// branch is proved against the main it branched from. Set to "" once the fix is merged.
|
|
Before string
|
|
// Package and Test are where the replay is run: a package of Repository for InRepository and Gate,
|
|
// of this module otherwise; Files are the files laid over the commit, from where the replay lives.
|
|
Package string
|
|
Test string
|
|
Files []string
|
|
// Home is where the replay's files live, when that is not the fix: the repository and ref.
|
|
Home, HomeRef string
|
|
// With is the other repositories the replay runs at their commits beside Repository's (Liveness).
|
|
With []Also
|
|
// Module is the directory of the Go module the replay runs in, within Repository, when that is not its
|
|
// root: a catalogue module is a Go module of its own. Package is relative to it; Files are not.
|
|
Module string
|
|
}
|
|
|
|
// Register is every replay, by incident.
|
|
var Register = []Replay{
|
|
{ID: "R236", Issue: 236, Kind: Gate, Repository: "mesh-controller", Fix: "feat/merge-gate",
|
|
Before: "origin/main",
|
|
What: "a login manager's service passed the catalogue check and was refused whole by the node-engine",
|
|
Asserts: "a manifest the node-engine refuses fails its pull request, naming the machine and the module",
|
|
Package: "./cmd/mesh-controller", Test: "TestIssue236", Files: []string{"cmd/mesh-controller/merge_gate_test.go"}},
|
|
{ID: "R262", Issue: 262, Kind: Resolver, Repository: "mesh-catalog", Fix: "20603b63e67323f2c725b242c0ea87281d4dee93",
|
|
What: "an Alpine container could not find a machine by its mesh name: NXDOMAIN for IPv6 is final to musl",
|
|
Asserts: "every machine's name resolves through the catalogue's resolver under musl and under glibc",
|
|
Package: ".", Test: "TestReplay262"},
|
|
{ID: "R263", Issue: 263, Kind: InRepository, Repository: "mesh-controller", Fix: "6d620f77c3f3f967be953ff760823c6a51b7b8d3",
|
|
What: "a consumer's 26-character identity refused the resolver holder's whole declaration",
|
|
Asserts: "a provider's machine composes whatever its consumers are called",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay263", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "feat/replays"},
|
|
{ID: "R266", Issue: 266, Kind: Bus, Repository: "mesh-catalog", Fix: "0275c2e",
|
|
What: "a merge on the bus was never handed to the controller: 2.10 skipped messages on a consumer with several filters",
|
|
Asserts: "a consumer filtered like the controller's is handed every followed message under the mesh's traffic",
|
|
Package: ".", Test: "TestReplay266"},
|
|
{ID: "R273", Issue: 273, Kind: InRepository, Repository: "mesh-controller", Fix: "8bfaf1523ed2dddde80daa0994f1044e59db44b1",
|
|
What: "a rule for the resolver re-bound a machine's database consumers to empty databases elsewhere",
|
|
Asserts: "a consumer beside its store stays bound to it while another machine holds the store's seat",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay273", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "feat/replays"},
|
|
// The controller's incidents of 2026-10-07, each a test laid over the merge before its fix. Home is
|
|
// the controller's commit that added them; it stays reachable once that pull request merges.
|
|
{ID: "R296", Issue: 296, Kind: InRepository, Repository: "mesh-controller", Fix: "3808634",
|
|
What: "a plan's clock restarted at every save: a build queued for minutes read \"building for 1s\", " +
|
|
"then 4s, then 9s, and was never LATE",
|
|
Asserts: "a plan's line counts from when it entered its tier, and a plan standing still in its tier is not " +
|
|
"saved again",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay296", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
|
{ID: "R299", Issue: 299, Kind: InRepository, Repository: "mesh-controller", Fix: "c37d774",
|
|
What: "a seat's first verbs, all optional, made the self-check say its holder silent on every machine",
|
|
Asserts: "a holder of a seat whose verbs are all optional is not said silent when it answers nothing",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay299", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
|
{ID: "R300", Issue: 300, Kind: InRepository, Repository: "mesh-controller", Fix: "e7dca6c",
|
|
What: "a merge that added a module said it changed nothing the mesh holds, and built nothing, so the " +
|
|
"module was never registered",
|
|
Asserts: "a merge adding a module asks the build seat for it, at the branch merged into, and opens no plan",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay300", Files: []string{"cmd/mesh-controller/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "7597294ff86383c171b8bec3920b5472fbce00e9"},
|
|
// A drill counted as a repair (2026-10-07): S15 read two deliberate drills, recorded through hand-act
|
|
// record for want of a verb, as a cause repaired twice. In a file of its own, since replays_test.go holds
|
|
// replays of later commits that do not build at the commit before this fix. Home is the controller's
|
|
// commit that added it; it stays reachable once that pull request merges.
|
|
{ID: "R292", Issue: 292, Kind: InRepository, Repository: "mesh-controller", Fix: "863ebd4",
|
|
What: "two drills of ADR 0240, each with the operator's word, recorded through hand-act record --cause drill, " +
|
|
"raised mesh.hand-acts.drill.healer-wanted",
|
|
Asserts: "a drill recorded in the hand-act log wants no healer, however it was recorded",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay292", Files: []string{"cmd/mesh-controller/replay292_test.go"},
|
|
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
|
// A seat's new verb deadlocked across two repositories (2026-10-07): the controller promising checks
|
|
// refused the delivery seat's holder that did not serve it, and a holder serving it was refused by the
|
|
// controller that did not promise it.
|
|
{ID: "R298", Issue: 298, Kind: InRepository, Repository: "mesh-controller", Fix: "f6aea4b",
|
|
What: "the delivery seat's new verb checks could land in neither repository first: each side's claim " +
|
|
"check refused the other's holder",
|
|
Asserts: "the delivery seat's holder holds it both before and after it serves checks",
|
|
Package: "./internal/catalogue", Test: "TestReplay298", Files: []string{"internal/catalogue/replays_test.go"},
|
|
Home: "mesh-controller", HomeRef: "80f9d26e6d4c60ec7d137110b9d84c841072f04e"},
|
|
// The push a recorded build waits for, counted as a repair (2026-10-07): a test in the controller, in
|
|
// the fix's own commit, recording two such pushes as the seat's push records them.
|
|
{ID: "R301", Issue: 301, Kind: InRepository, Repository: "mesh-controller", Fix: "e92a3fe",
|
|
What: "the operator's pushes of new builds whose upgrade policy is record — the resolver, the network " +
|
|
"managers, the packet filter — were counted as repairs, and S15 wanted a healer for each cause",
|
|
Asserts: "a push that moves only recorded builds, recorded as the push records it, wants no healer " +
|
|
"however often its cause is given",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay301", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
|
// The lab's Go replays never compiled before a lab merge (2026-10-07): the build agent ran a repository's
|
|
// check in one toolchain. Before the fix there was no part to run, so the commit before is held to fail
|
|
// by not having it.
|
|
{ID: "R302", Issue: 302, Kind: Gate, Repository: "mesh-controller", Fix: "98ef7ba",
|
|
What: "mesh-lab's replays register merged with NOT CHECKED HERE: its check ran in the TypeScript " +
|
|
"toolchain, which holds no Go compiler",
|
|
Asserts: "a repository in two languages has each declared part of its own check run in that part's " +
|
|
"toolchain, and the layer passes only when every part does",
|
|
Package: "./internal/builder", Test: "TestARepositoryInTwoLanguagesIsCheckedInBoth",
|
|
Files: []string{"internal/builder/check_test.go"}},
|
|
// A recheck left the old verdict standing (2026-10-07): the forge keeps the newest status of each context
|
|
// on a head, and a recheck asks of the same head, so the old green let the pull request merge while the
|
|
// fresh check ran. A test in mesh-delivery, which asks the forge's holder (core) to put the merge check
|
|
// back to pending; the holder's own half is its TypeScript test.
|
|
{ID: "R305", Issue: 305, Kind: InRepository, Repository: "mesh-catalog", Fix: "74f912e",
|
|
What: "a pull request rechecked at 23:04 kept mesh/merge-gate and mesh/repo-check success from 16:59, " +
|
|
"and the forge would have merged it while its fresh check ran",
|
|
Asserts: "a recheck puts the head's merge check statuses back to pending before it asks the check, and " +
|
|
"is refused whole when the forge cannot be told",
|
|
Module: "modules/mesh-delivery", Package: "./cmd/mesh-delivery", Test: "TestARecheck",
|
|
Files: []string{"modules/mesh-delivery/cmd/mesh-delivery/recheck_test.go",
|
|
"modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go"}},
|
|
// A pull request ran its own copy of the repository's check (2026-10-08): the build agent read
|
|
// merge-check.sh from the change's head, so a branch cut before the script, or one that deleted or
|
|
// gutted it, merged with none of its tests run. Written with only what the agent had before the fix, so
|
|
// it fails on the commit before by answering the warning. The fix is still on its branch: proved
|
|
// against the main it branched from.
|
|
{ID: "R310", Issue: 310, Kind: InRepository, Repository: "mesh-controller", Fix: "cb0327d",
|
|
Before: "1a50d6e",
|
|
What: "the media catalogue's Lidarr pull request, cut before its repository had a merge-check.sh, was " +
|
|
"answered no repository check defined as a warning and merged with none of its tests run",
|
|
Asserts: "where the base branch holds a merge-check.sh, its script judges the change's tree, and a head " +
|
|
"without one is never answered the warning",
|
|
Package: "./internal/builder", Test: "TestReplay310", Files: []string{"internal/builder/replay310_test.go"}},
|
|
// Two order rules ordered one pair both ways (2026-10-08): the controller's member moved the build agent,
|
|
// which builds the node-engine, and the node-engine goes before the controller, so the group
|
|
// feat/a-machine-joins-through-the-tunnel was refused as a cycle and merged by hand. A test in the
|
|
// controller, using only what orderOf had before its fix, laid over the commit before it.
|
|
{ID: "R309", Issue: 309, Kind: InRepository, Repository: "mesh-controller", Fix: "c5a2edf",
|
|
What: "a delivery group of the controller, the node-engine and the lab, each ready, was rejected: built by " +
|
|
"put the controller's member first and engine before controller put the node-engine's first",
|
|
Asserts: "rules that order one pair both ways are resolved by precedence — built by over engine before " +
|
|
"controller — and the group is ordered, the controller's member first",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay309", Files: []string{"cmd/mesh-controller/replay309_test.go"}},
|
|
// An answer larger than one message of the bus (2026-10-08): the controller's conditions and status
|
|
// outgrew the bus's max_payload once 689 stalled deliveries were open conditions, the client library
|
|
// refused every reply inside the controller, and every caller timed out while calls said answered.
|
|
{ID: "R314", Issue: 314, Kind: InRepository, Repository: "mesh-controller", Fix: "175b28e",
|
|
What: "conditions and status answered three times what one message of the bus carries; the reply was " +
|
|
"refused inside the controller and every caller waited 30 s and read that it did not answer",
|
|
Asserts: "an answer larger than one message arrives whole to a caller that pages, and one that does not " +
|
|
"is told so in a message that fits — never a timeout",
|
|
Package: "./internal/link", Test: "TestReplay314", Files: []string{"internal/link/replay314_test.go"}},
|
|
// One module's wait for a new login held every other module's walk (2026-10-08): the walk of the builds
|
|
// waiting for a gate sent the laptop three moves in one send, the lighting module's account said relogin
|
|
// needed and its daemon's unit failed in that account's manager, and the gate failed the send at its bound,
|
|
// put the module back (taking its account group back) and stopped every walk behind it. A test in the
|
|
// controller, reading the statements as the node-engine's JSON, so it is laid over the commit before.
|
|
// Fix is the head of the fix: e3b5c22 alone raises the new condition kind without the plain words the
|
|
// controller's suite requires since hq ADR 0253, which 51d7bbf gives it. Before is the main both stood on.
|
|
{ID: "R318", Issue: 318, Kind: InRepository, Repository: "mesh-controller", Fix: "51d7bbf", Before: "6df30b6",
|
|
What: "a send of three modules to the laptop failed its gate after ten minutes because one module's account " +
|
|
"needed a new login; the module was put back and the walks of the other modules stopped behind it",
|
|
Asserts: "a wait for a person's new login passes the gate with the wait carried and said as the module's " +
|
|
"relogin-needed condition; nothing is put back, and the walk goes on to the next machine",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay318", Files: []string{"cmd/mesh-controller/replay318_test.go"}},
|
|
// An assignment answered as unknown while its module's build ran (2026-10-08): the merge adding `sensors`
|
|
// asked for its build (issue 300), and `assign g14 sensors` a minute later said "no module of that name",
|
|
// which read as a module nobody registered. A test in the controller, written with only what it had
|
|
// before the fix, laid over the commit before. The fix is still on its branch: Fix is its pull request's
|
|
// head, proved against the main it is rebased on.
|
|
{ID: "R325", Issue: 325, Kind: InRepository, Repository: "mesh-controller", Fix: "8adb7f1", Before: "fe00fec",
|
|
What: "a minute after the merge that added sensors asked for its build, assign g14 sensors answered no " +
|
|
"module of that name, and the same call worked minutes later",
|
|
Asserts: "an assignment made while its module's build runs says the build, is kept, and is made when the " +
|
|
"build registers the module",
|
|
Package: "./cmd/mesh-controller", Test: "TestReplay325", Files: []string{"cmd/mesh-controller/replay325_test.go"}},
|
|
// A declared tool check never asked on a live machine (2026-10-08): the queue reached the bus through an
|
|
// assertion only OverNATS met, and the NATS link the node-engine attaches had no Ask, so every tool check
|
|
// read "no link to the bus is open" while the link was up. A test in the node-engine, written with only
|
|
// what it had before the fix; Home is its commit that added it.
|
|
{ID: "R331", Issue: 331, Kind: InRepository, Repository: "mesh-host", Fix: "824911d",
|
|
What: "the sensors module's tool check on the laptop read no link to the bus is open for hours while the " +
|
|
"node-engine's link was up and the same tool answered through the mesh MCP server",
|
|
Asserts: "a tool check on the NATS link the node-engine attaches reaches that link, and is not refused as no link",
|
|
Package: "./internal/link", Test: "TestReplay331", Files: []string{"internal/link/replay331_test.go"},
|
|
Home: "mesh-host", HomeRef: "5684a4579d92"},
|
|
// Not a core incident, and the first of its kind: a module's. The crash loop was found by a person
|
|
// reading the agent server's log for another reason (issue 268); research 032 measured it, and ADR
|
|
// 0240 makes the node-engine judge it and the gate fail it.
|
|
{ID: "R-crashloop", Issue: 268, Kind: Liveness, Repository: "mesh-controller",
|
|
Fix: "9d15f3a",
|
|
With: []Also{{Repository: "mesh-host", Fix: "03031a4"}},
|
|
What: "the agent server crash-looped about a hundred times while the mesh read it applied, its tools served " +
|
|
"and nothing raised: the gate judged a module by what the mesh saw from outside",
|
|
Asserts: "a container whose program exits at start is said unhealthy by the node-engine, and its build fails " +
|
|
"its gate on the first machine and is put back there",
|
|
Package: ".", Test: "TestReplayCrashLoop", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
|
// A module's again: the silent web application, which no liveness and no TCP check sees, raised by the
|
|
// HTTP check its module declares (ADR 0240 Phase B).
|
|
{ID: "R145", Issue: 145, Kind: Liveness, Repository: "mesh-controller",
|
|
Fix: "7f25666",
|
|
With: []Also{{Repository: "mesh-host", Fix: "56e2ebe"}},
|
|
What: "a web application accepted TCP and answered nothing for eleven hours while the mesh said its machine " +
|
|
"healthy: its port was open and its program ran",
|
|
Asserts: "a web server whose application never answers is said unhealthy by its declared HTTP check within two " +
|
|
"looks, and the controller raises the module's condition on the second",
|
|
Package: ".", Test: "TestReplaySilentWebApp", Files: []string{"cmd/mesh-controller/replays_test.go"}},
|
|
}
|
|
|
|
// Find is the replay of that id, or of that issue.
|
|
func Find(id string) (Replay, bool) {
|
|
for _, r := range Register {
|
|
if r.ID == id || r.ID == "R"+id {
|
|
return r, true
|
|
}
|
|
}
|
|
return Replay{}, false
|
|
}
|