Compare commits
107
Commits
0042ca9258
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
51ef3eb7e2 | ||
|
|
346e613995 | ||
|
|
25ca9898d5 | ||
|
|
709c095387 | ||
|
|
c262de3833 | ||
|
|
a815433214 | ||
|
|
61dc90e508 | ||
|
|
a7cf5c0c1b | ||
|
|
c6f86ae935 | ||
|
|
1aeb4fe8d8 | ||
|
|
08108b569d | ||
|
|
14ff89fa40 | ||
|
|
2126e7b2cb | ||
|
|
dcdfcf104e | ||
|
|
d23ace1646 | ||
|
|
5dbde0b13a | ||
|
|
db3868e2b0 | ||
|
|
05039c4f10 | ||
|
|
b7bf601ca4 | ||
|
|
bff32e3370 | ||
|
|
d6b62387f2 | ||
|
|
4789624857 | ||
|
|
e00862e317 | ||
|
|
3a92e4b80c | ||
|
|
bfddf78bf3 | ||
|
|
0b291d89f3 | ||
|
|
a924efcc28 | ||
|
|
b421c73a5a | ||
|
|
017b1401e4 | ||
|
|
476cda417d | ||
|
|
74ba3ff1d4 | ||
|
|
891c7a945e | ||
|
|
83cbeb8db8 | ||
|
|
ab6db9369b | ||
|
|
84571b4825 | ||
|
|
d57196102d | ||
|
|
e6402cf777 | ||
|
|
4f0d144833 | ||
|
|
98d94ef71e | ||
|
|
4e13280604 | ||
|
|
8783a13448 | ||
|
|
a31cfcf461 | ||
|
|
75b3861911 | ||
|
|
694555214a | ||
|
|
a7249541df | ||
|
|
95d8253f71 | ||
|
|
784b487bf9 | ||
|
|
7ae711ba0b | ||
|
|
51739c4302 | ||
|
|
5cac268457 | ||
|
|
ce6ae943b7 | ||
|
|
9ef9830dcf | ||
|
|
dfac01a6dd | ||
|
|
51f5ce5c0f | ||
|
|
fc64a2c1a4 | ||
|
|
9fc4e74b0d | ||
|
|
225dfa9451 | ||
|
|
bfefb1dbdb | ||
|
|
85c0a3e567 | ||
|
|
a39765c924 | ||
|
|
a8921fe737 | ||
|
|
c8f430290e | ||
|
|
d98d6fca11 | ||
|
|
b8cdfce16d | ||
|
|
c4a8455e2e | ||
|
|
bf7297b7ed | ||
|
|
1bb0ef5658 | ||
|
|
5a9917f802 | ||
|
|
8a6ee9177c | ||
|
|
dd577e9ebe | ||
|
|
a9b8f41570 | ||
|
|
92a5e8fc05 | ||
|
|
8c91ba1cfa | ||
|
|
0f7f628730 | ||
|
|
970da74136 | ||
|
|
4d4012cdf6 | ||
|
|
0a61531c42 | ||
|
|
4b0b659084 | ||
|
|
f555d523c7 | ||
|
|
4f93d304d7 | ||
|
|
d898bd87e8 | ||
|
|
7e4da874a9 | ||
|
|
53092020eb | ||
|
|
00817fb9e3 | ||
|
|
9946e852e1 | ||
|
|
9f6aa7ea9c | ||
|
|
672c994afa | ||
|
|
2f9bb73685 | ||
|
|
5c193b3f54 | ||
|
|
fbf9440e8e | ||
|
|
9510bf5311 | ||
|
|
d940e14ec8 | ||
|
|
80456981be | ||
|
|
d2ed3152d3 | ||
|
|
24d99ddd24 | ||
|
|
b759e36bfd | ||
|
|
0b8e84334f | ||
|
|
39c802cbd4 | ||
|
|
86a084b7ff | ||
|
|
85f972749a | ||
|
|
7abb268de6 | ||
|
|
78a2274baf | ||
|
|
3f9b316015 | ||
|
|
e05825a881 | ||
|
|
7b4916e9ec | ||
|
|
b5b68e8852 | ||
|
|
814c9e563f |
@@ -164,6 +164,12 @@ def check_rests_on(failures, records):
|
||||
# decision is exactly what as-is is for."
|
||||
if rel(path).startswith("03-DESIGN/00-as-is/"):
|
||||
continue
|
||||
# A withdrawn record's citations are history. It instructs nobody -- every reader
|
||||
# is sent to its superseder -- so what it was built on may itself be withdrawn.
|
||||
# Refusing that would mean rewriting the lineage of a record whose reasoning is
|
||||
# the thing the immutability rule protects.
|
||||
if frontmatter(read(path)).get("status") == "superseded":
|
||||
continue
|
||||
# An extension that supersedes legitimately names what it replaced.
|
||||
this = ADR_FILE.match(os.path.basename(path))
|
||||
supersedes = records[number]["front"].get("superseded-by", "")
|
||||
@@ -241,13 +247,20 @@ def check_supersession_symmetry(failures, records):
|
||||
failures.add("supersession", rel(record["path"]), f"superseder does not exist: {by}")
|
||||
continue
|
||||
other = records[match.group(1)]
|
||||
claims = os.path.basename(str(other["front"].get("supersedes", "")))
|
||||
if claims != record["name"]:
|
||||
# `supersedes:` may name one record or several. One decision replacing two is a real
|
||||
# situation -- two records that built and refined the same wrong mechanism are withdrawn
|
||||
# by the one record that removes it -- and a check that allows only one would force
|
||||
# either a chain of pro-forma records or an unmarked supersession.
|
||||
claimed = other["front"].get("supersedes", "")
|
||||
if isinstance(claimed, str):
|
||||
claimed = [claimed] if claimed else []
|
||||
claims = [os.path.basename(str(entry)) for entry in claimed]
|
||||
if record["name"] not in claims:
|
||||
failures.add(
|
||||
"supersession",
|
||||
rel(other["path"]),
|
||||
f"ADR {number} says this supersedes it; this record does not say so "
|
||||
f"(supersedes: {claims or 'absent'})",
|
||||
f"(supersedes: {', '.join(claims) or 'absent'})",
|
||||
)
|
||||
|
||||
|
||||
@@ -299,8 +312,13 @@ def check_progressive_insights(failures, records):
|
||||
unmarked change stands out as the anomaly it is.
|
||||
"""
|
||||
phrase = re.compile(r"progressive insight", re.I)
|
||||
marker = re.compile(r"\*\*Progressive insights?\s*[\u2014\u2013-]\s*(\d{4}-\d{2}-\d{2})\.?\*\*")
|
||||
loose = re.compile(r"\*\*[^*]*[Pp]rogressive insights?[^*]*\*\*")
|
||||
# Both patterns stay on one line: a bold run does not span paragraphs, and `[^*]*` across
|
||||
# newlines will happily join an unrelated `**` far above to the marker below, reporting the
|
||||
# whole span between them. It did exactly that the first time this ran.
|
||||
# Trailing words after the date are allowed — "— 2026-09-26, correcting the one above." — so
|
||||
# an insight can say what it relates to. Only the date's presence and position are fixed.
|
||||
marker = re.compile(r"\*\*Progressive insights?[ \t]*[\u2014\u2013-][ \t]*(\d{4}-\d{2}-\d{2})[^*\n]*\*\*")
|
||||
loose = re.compile(r"\*\*[^*\n]*[Pp]rogressive insights?[^*\n]*\*\*")
|
||||
iso = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
||||
|
||||
for number, record in sorted(records.items()):
|
||||
@@ -313,6 +331,10 @@ def check_progressive_insights(failures, records):
|
||||
for m in loose.finditer(text):
|
||||
if any(s <= m.start() and m.end() <= e for s, e, _ in good):
|
||||
continue
|
||||
# A bold run carrying a link is discussing an insight — usually another record's —
|
||||
# rather than marking one. A marker never needs to cite anything.
|
||||
if "](" in m.group(0):
|
||||
continue
|
||||
failures.add("insights", rel(record["path"]),
|
||||
"a progressive insight is not in the dated marked form "
|
||||
"'**Progressive insight \u2014 YYYY-MM-DD.**': %s" % m.group(0))
|
||||
@@ -328,7 +350,15 @@ def check_progressive_insights(failures, records):
|
||||
if any(s <= m.start() and m.end() <= e for s, e in covered):
|
||||
continue
|
||||
line = text.rfind("\n", 0, m.start()) + 1
|
||||
if text[line:m.start()].lstrip().startswith("#"):
|
||||
end = text.find("\n", m.end())
|
||||
whole = text[line:end if end != -1 else len(text)]
|
||||
if whole.lstrip().startswith("#"):
|
||||
continue
|
||||
# A line that also carries a link is discussing the rule, not marking a correction:
|
||||
# a marker never needs to cite anything, and a record that reasons about the policy
|
||||
# must be able to name it. Bare prose with no citation is the informal marking this
|
||||
# is here to catch.
|
||||
if "](" in whole:
|
||||
continue
|
||||
if loose.search(text, line, text.find("\n", m.end()) + 1 or len(text)):
|
||||
continue
|
||||
|
||||
+17
-3
@@ -31,8 +31,22 @@ another — and a mesh you cannot name precisely is a mesh two people describe d
|
||||
- **store** — the one postgres server. It holds the controller's own context databases
|
||||
(`inventory`, `identity`, `licences` — a context owns its store, [ADR 0008](../02-DECISIONS/0008-a-context-owns-its-store.md))
|
||||
and every module's own database. One server, many databases — never one shared "mesh database".
|
||||
- **broker** — the one lavinmq message bus. It carries the mesh bus on the `/` vhost and a vhost per
|
||||
consumer that requires `amqp`.
|
||||
- **bus** — the mesh's own nervous system: NATS, one per mesh, carrying every link the mesh has —
|
||||
control, declarations, builds, events, tool calls
|
||||
([ADR 0106](../02-DECISIONS/0106-the-bus-is-nats.md)). A module reaches it by requiring
|
||||
`mesh-bus` ([ADR 0128](../02-DECISIONS/0128-the-mesh-bus-is-required-not-ambient.md)); one that
|
||||
does not require it has no account on it. Held by the `mesh-broker` seat, which is named after
|
||||
the *role* rather than the server, so the server can change without the seat doing so.
|
||||
- **the deprecated broker** — the lavinmq module. It was the mesh's bus and is not any more. It
|
||||
keeps running as an **ordinary provider** of the `amqp` provision, for modules that need a
|
||||
message broker of their own the way something needs a database
|
||||
([ADR 0127](../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md))) — no seat, not foundation,
|
||||
never raised at genesis, and a mesh that never installs it is complete.
|
||||
|
||||
Say *the deprecated broker*, not "the compatibility broker" (it serves the mesh's own modules,
|
||||
not only the predecessor's) and not "the AMQP broker" (naming it after a protocol invites
|
||||
describing the bus by contrast with it, which is backwards: the bus is the mesh's nervous
|
||||
system and this is a module).
|
||||
|
||||
## What the mesh stores and serves
|
||||
|
||||
@@ -47,7 +61,7 @@ another — and a mesh you cannot name precisely is a mesh two people describe d
|
||||
- **seat** — a named role at a scope (node / site / mesh), held by a module assignment, from a
|
||||
**closed set** the mesh defines: a claim naming a seat outside the set is refused. A seat may
|
||||
**deliver a provision**, and its holder is then the mesh's answer for it when several modules
|
||||
provide it ([ADR 0110](../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)).
|
||||
provide it ([ADR 0126](../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md))).
|
||||
The set, with who holds each seat, is the overview of what a mesh has
|
||||
([26 — The seats](../03-DESIGN/01-to-be/26-the-seats.md)). A seat has a **capacity**: a
|
||||
capacity-1 seat is exclusive (one holder); a higher-capacity seat is a **bench** (several holders
|
||||
|
||||
@@ -76,6 +76,22 @@ three relationships, one broker, one runtime, all declared on the manifest.
|
||||
- The runtime must dispatch a module's event handlers as well as its tools; that generalisation is
|
||||
small (both arrive by importing the module's entrypoint) but it is real work.
|
||||
|
||||
## Progressive insight
|
||||
|
||||
> **Progressive insight — 2026-09-26.** *"No provisioner and no per-consumer setup" was a fact
|
||||
> about the transport, and the transport changed.* This record's table says an event's machinery is
|
||||
> "nothing but the broker's topic routing", and the text that an event needs "no per-consumer setup
|
||||
> — only a subscription". That was true of a topic exchange, where a binding cost nothing and the
|
||||
> broker fanned out. On NATS
|
||||
> ([ADR 0106](0106-the-bus-is-nats.md)) a subscription is a **durable consumer**: a real object
|
||||
> with a name, an ack policy, a delivery limit and its own ack subject, created when a module is
|
||||
> assigned and removed when it is not. Per-consumer setup exists, and the controller does it.
|
||||
>
|
||||
> The decision is untouched — events are declared on both sides, 1:many, credential-free, and
|
||||
> still provisioning's lighter sibling; the lightness is now relative rather than absolute.
|
||||
> [ADR 0126](0126-a-module-declares-its-own-seats.md) adds the relationship this record's two
|
||||
> columns had no room for: work addressed to a role, where exactly one holder must act.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0002](0002-nodes-communicate-over-a-broker.md) — the broker events ride.
|
||||
|
||||
@@ -75,6 +75,30 @@ after its deliveries are exhausted; a module's account cannot publish outside it
|
||||
subscribe outside its `consumes`. Then the cutover bed: a mesh on AMQP with the predecessor's
|
||||
compatibility broker beside it moves its bus in one rollout with every node reporting afterwards.
|
||||
|
||||
## Progressive insight
|
||||
|
||||
> **Progressive insight — 2026-09-26.** *The compatibility broker was not single-purpose when this
|
||||
> was written.* This record says the adopted AMQP broker is "kept as a module with one purpose —
|
||||
> the predecessor's clients". Two modules of the new mesh also depended on it, through a `requires:
|
||||
> ["amqp"]` grant its provisioner answered with a private vhost — `amqp-ping` and
|
||||
> `amqp-email-forwarder`. On the retirement condition below, both would have been left requiring
|
||||
> something no provider answers.
|
||||
> [ADR 0125](0125-the-bus-is-the-only-broker.md) resolves it by moving them onto the bus and
|
||||
> retiring the interface, which makes this record's sentence true rather than merely intended. The
|
||||
> decision — the bus is NATS, the AMQP broker becomes the predecessor's compatibility broker and
|
||||
> retires with the last of them — is unchanged.
|
||||
|
||||
> **Progressive insight — 2026-09-26, correcting the one above.** *The broker is not a
|
||||
> compatibility module at all, and the sentence does not become true.* The insight above said
|
||||
> [ADR 0125](0125-the-bus-is-the-only-broker.md) would make "one purpose — the predecessor's
|
||||
> clients" true by moving the mesh's own modules off it.
|
||||
> [ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) supersedes that: nothing moves off, because
|
||||
> a module may legitimately need an AMQP broker as a backing service the way it needs a database.
|
||||
> The broker becomes **an ordinary provider module** — no seat, not foundation, not raised at
|
||||
> genesis, and with no retirement condition, because the day its last client disappears is not a
|
||||
> day anything is waiting for. What this record decided — the mesh's bus is NATS — is untouched
|
||||
> by both; what was wrong was the sentence describing what happens to the old server, twice.
|
||||
|
||||
## References
|
||||
|
||||
- [research 014](../01-RESEARCH/014-the-bus-on-nats/00-overview.md)
|
||||
|
||||
@@ -9,6 +9,17 @@ extends: 0009-modules-and-the-graph.md
|
||||
|
||||
# 110. A seat is held by one assignment, from a closed set, and it may deliver a provision
|
||||
|
||||
> **Narrowed, not replaced — 2026-09-27, on merging two lines of work.** This was marked superseded by
|
||||
> [ADR 0126](0126-a-module-declares-its-own-seats.md), and that overstated it: 0126 says in as many
|
||||
> words that *"everything 0110 decided about what a seat is stands untouched"*. What moved is where the
|
||||
> set lives and who may add to it —
|
||||
> [0126](0126-a-module-declares-its-own-seats.md) lets a module declare one and makes the set derived,
|
||||
> [0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md) names the mesh's own
|
||||
> for their scope, and [0122](0122-a-seat-is-data-a-rename-is-a-database-update.md) moves them out of
|
||||
> code into a table. **What a seat *is* — one holder at its scope, a definition saying what a module
|
||||
> can hold against an assignment saying what it does, a role made singular rather than a module — is
|
||||
> this record and still current**, which is why those three rest on it.
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0009](0009-modules-and-the-graph.md) introduced claims: a module declares something
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: proposed
|
||||
status: accepted
|
||||
date: 2026-09-25
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
|
||||
@@ -40,7 +40,7 @@ undeclare can do. Found reviewing the uplink modules
|
||||
- the uplink modules would have stopped the network manager, taking the machine off the only
|
||||
link the mesh reaches it by.
|
||||
|
||||
[ADR 0117](0117-a-machines-uplink-is-a-seat.md) answered that for its own modules with a
|
||||
[ADR 0125](0117-a-machines-uplink-is-a-seat.md) answered that for its own modules with a
|
||||
service declared with no `state`. Every other module that declares a unit it did not make is
|
||||
exposed in the same way, and relying on each author to remember an opt-out is how the next one
|
||||
is missed.
|
||||
@@ -91,7 +91,7 @@ records the state it first found the unit in, and undeclaring returns the unit t
|
||||
- The service's settings the mesh wrote are given back by their own resources (a kept original
|
||||
restored, a region or keys removed). A running service keeps running on what it read until it
|
||||
next reads its configuration; the mesh does not restart it to make it notice.
|
||||
- A service declared with no `state` (ADR 0117) remains the way to say the mesh must not
|
||||
- A service declared with no `state` (ADR 0125) remains the way to say the mesh must not
|
||||
**start** a unit either; undeclared, it is forgotten.
|
||||
|
||||
## Consequences
|
||||
@@ -113,7 +113,7 @@ records the state it first found the unit in, and undeclaring returns the unit t
|
||||
## References
|
||||
|
||||
- [issue 130](../04-ISSUES/130-undeclaring-a-service-stops-it/00-report.md): the finding
|
||||
- [ADR 0117](0117-a-machines-uplink-is-a-seat.md): the uplink modules, and a service with no state
|
||||
- [ADR 0125](0117-a-machines-uplink-is-a-seat.md): the uplink modules, and a service with no state
|
||||
- [ADR 0102](0102-the-mesh-writes-into-a-shared-file-never-over-it.md), [ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md):
|
||||
what is given back, and how
|
||||
- mesh-host `internal/apply/apply.go` (`remove`, the service case)
|
||||
|
||||
@@ -53,7 +53,7 @@ is removed from where its unit reads it.**
|
||||
reporting it.
|
||||
- **The mesh never brings it back.** Undeclaring the private network does not restore the found
|
||||
tunnel: the mesh stopped it, and nothing is started on the way out
|
||||
([ADR 0118](0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md)). A machine whose
|
||||
([ADR 0126](0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md)). A machine whose
|
||||
private network is unassigned has no tunnel until it is assigned again — which is what
|
||||
unassigning it means.
|
||||
|
||||
@@ -83,5 +83,5 @@ is removed from where its unit reads it.**
|
||||
- [ADR 0105](0105-the-mesh-adopts-the-predecessors-tunnel-in-place.md): the take, and why it keeps
|
||||
the found configuration during it
|
||||
- [ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md): kept originals
|
||||
- [ADR 0118](0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md): nothing is started on the way out
|
||||
- [ADR 0126](0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md): nothing is started on the way out
|
||||
- mesh-host `internal/apply/takeover.go`
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 0112-a-module-definition-names-no-node-mesh-or-path.md
|
||||
---
|
||||
|
||||
# 120. A roster fact carries its format as a template: the mesh owns the data, the module owns the format
|
||||
|
||||
## Context
|
||||
|
||||
A **fact** is a thing only the mesh knows — which machines exist, what they are called, where they
|
||||
are — written into a file where a module asks for it. The mesh computes it from the graph; a module
|
||||
loads it, restarts on it, does what its software does with it. Facts replaced three modules that
|
||||
existed only because computed output needed somewhere to live and ran no software of their own
|
||||
([ADR 0040](0040-what-a-module-is.md)).
|
||||
|
||||
But the *format* lived in the control plane. A fact was a name from a closed list, and each name had
|
||||
a formatter written in Go beside the others: `node-names` wrote the roster as an `/etc/hosts` file,
|
||||
`node-zones` wrote it as a dnsmasq resolver's `local=`/`address=` lines. Adding a consumer meant
|
||||
adding a formatter — in the consumer's own configuration language — to the mesh.
|
||||
|
||||
The ssh work made the cost plain. An operator's `~/.ssh` wants three roster projections — a
|
||||
`known_hosts`, an ssh `config` of `Host` blocks, an `authorized_keys` — each in ssh's syntax. Under
|
||||
the closed list that is three more formatters in the control plane, teaching it ssh's configuration
|
||||
language. And it does not stop at ssh: every daemon that reads the roster in its own file format
|
||||
would put its grammar here. The control plane was accreting the configuration languages of software
|
||||
it does not run — the exact thing [ADR 0040](0040-what-a-module-is.md) says is a
|
||||
module's and not the mesh's.
|
||||
|
||||
The shape underneath is one shape. WireGuard's `[Peer]` blocks, `/etc/hosts`, dnsmasq's zones, an
|
||||
ssh `known_hosts` — all of them are *the roster, projected into a file*. Only the projection differs,
|
||||
and the projection belongs to whoever runs the software that reads it.
|
||||
|
||||
## Considered Options
|
||||
|
||||
**1. Keep the closed list; add a formatter per consumer.** Rejected. The control plane learns the
|
||||
configuration language of every daemon any module might run, without bound, and each format lives in
|
||||
the mesh rather than in the module that owns the file. A module cannot change how its own file is
|
||||
written without a control-plane change.
|
||||
|
||||
**2. A general placeholder vocabulary over `content`, like `${machine:address}` but for the
|
||||
roster.** Rejected. The mesh's other substitutions each resolve to *one* scalar — this machine's
|
||||
address, one provider's port. The roster is inherently a *repetition*: one block per machine. A flat
|
||||
`${…}` vocabulary cannot iterate, and a mechanism that could would be a template in all but name.
|
||||
|
||||
**3. The module gives a path and a template over the roster; the mesh renders it.** Chosen. The mesh
|
||||
owns the data — who exists, their names and addresses — and hands it to a Go `text/template` the
|
||||
module wrote. The mesh renders and reads neither the template's intent nor the file's meaning.
|
||||
|
||||
## Decision
|
||||
|
||||
**A fact is a path and a template.** In a module's manifest, `facts` maps a name the module chooses
|
||||
to a `{ path, template }`. The template is a Go `text/template` over a fixed **roster view**:
|
||||
|
||||
- `.Node` — this machine's bare name.
|
||||
- `.Suffix` — what a mesh name ends in (`internal`, or the operator's choice), as composed.
|
||||
- `.Names` — every name the mesh serves: the machines *and* the names it was told to route.
|
||||
- `.Machines` — only the machines that are nodes of this mesh.
|
||||
|
||||
Each of `.Names` and `.Machines` is a list of `{ Name, FQDN, Address }`. A machine the mesh has a
|
||||
record for but cannot yet place has no address and is left out of both — a name that resolves to
|
||||
nothing is a connection that hangs, so it is omitted rather than written (the same rule as before).
|
||||
|
||||
**The mesh owns the data; the module owns the format.** The control plane holds **no** formatter.
|
||||
The two built-in projections render through the same path any module uses:
|
||||
|
||||
- **`/etc/hosts`** is a template on the mesh's own network module. The mesh writes `/etc/hosts`
|
||||
because being on the private network is what gives a machine a name — but the *layout* is a
|
||||
template like any other, shipped with the control plane because that module ships with it, not
|
||||
because the control plane knows the hosts-file format.
|
||||
- **dnsmasq's zones** move into dnsmasq. The `local=`/`address=` grammar is dnsmasq's configuration
|
||||
language, and it now lives in dnsmasq's manifest, where the module that runs dnsmasq owns it.
|
||||
|
||||
**A fact also says whether its file is the mesh's whole or a region of the machine's.** A hosts file
|
||||
is the machine's — its `localhost`, the operator's lines, another tool's marked blocks — so
|
||||
`node-names` is `shared`: the mesh owns only its region and keeps the rest byte for byte, the host
|
||||
laying it down `into: block`
|
||||
([ADR 0102](0102-the-mesh-writes-into-a-shared-file-never-over-it.md), hq issue 128). A resolver's
|
||||
zones file is the mesh's whole, and is not shared. The template renders the content either way;
|
||||
`shared` decides how the host writes it. This composes with hq 128 rather than replacing it: the
|
||||
region *mechanism* is the host's, the region's *format* is the module's template.
|
||||
|
||||
**The names-vs-machines distinction is the template's choice** ([04-ISSUES/111](../04-ISSUES/111-the-resolver-is-told-names-the-mesh-serves-not-only-machines/00-report.md)):
|
||||
a container's hosts ranges `.Names`, so a routed name resolves to the machine serving it; a resolver
|
||||
told the mesh's suffix is its own ranges `.Machines`, or a routed name written there with the suffix
|
||||
appended is a name nobody will ever ask for.
|
||||
|
||||
**A template that will not render is refused at composition, not on a machine.** A template that does
|
||||
not parse, or reads a field the roster does not have, fails where the manifest is — the closed-list
|
||||
safety, moved from the fact's *name* to the roster's *shape*. A daemon that starts, reads a file the
|
||||
mesh could not render, and answers nothing is a much worse way to find out.
|
||||
|
||||
**WireGuard stays a computed generator, and that is the line.** Its `mesh0.conf` is not a pure roster
|
||||
projection — it carries topology the control plane decides: which peers are reachable, endpoints, hub
|
||||
forwarding, keepalive for a NAT'd node. And it is *foundational*: the overlay must be up before any
|
||||
module can be delivered, so the thing that writes it cannot itself be a delivered module. The line
|
||||
this draws: **the substrate that delivery rides on is the control plane's; everything layered on a
|
||||
working overlay is a roster template.** DNS, hosts, and ssh are layered; the overlay is the floor.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **ssh is two templates and no control-plane change.** Once the roster view carries a machine's ssh
|
||||
host key and its operator account ([to-be 29](../03-DESIGN/01-to-be/29-a-node-has-operator-accounts.md)),
|
||||
`known_hosts`, the ssh `config`, and `authorized_keys` are templates on the ssh modules — the mesh
|
||||
gains no knowledge of ssh's syntax. This ADR is what makes that work land without touching the
|
||||
controller.
|
||||
- **A new roster projection never touches the control plane.** Any module that reads the roster in
|
||||
its own format ships its own template.
|
||||
- **A module can change how its own file is written** without a control-plane change — it is editing
|
||||
its own manifest.
|
||||
- **The schema changed and is not backward compatible.** A fact was a string (a path); it is now
|
||||
`{ path, template }`. The old string form has no template and cannot be auto-upgraded, because the
|
||||
format it implied was the formatter this ADR deletes. The controller and every catalogue module
|
||||
using facts — only dnsmasq — land together. A controller and a catalogue that disagree cannot
|
||||
compose the module: the running daemon on a machine is unaffected, but the mesh will not send it a
|
||||
new declaration until both sides agree.
|
||||
- **The output did not change.** The `/etc/hosts` and dnsmasq zones a machine receives are
|
||||
byte-for-byte what the deleted formatters wrote, pinned by tests that render the built-in template
|
||||
and compose the real dnsmasq manifest.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0040](0040-what-a-module-is.md): a module is software the mesh runs — a
|
||||
format the mesh knows for software it does not run was the accretion this stops
|
||||
- [ADR 0112](0112-a-module-definition-names-no-node-mesh-or-path.md): a module definition names no
|
||||
path; this is its sibling for content — a module definition names no format the mesh must know
|
||||
- [to-be 29](../03-DESIGN/01-to-be/29-a-node-has-operator-accounts.md): the ssh consumer this
|
||||
unblocks, and the roster fields it will add
|
||||
- [04-ISSUES/111](../04-ISSUES/111-the-resolver-is-told-names-the-mesh-serves-not-only-machines/00-report.md): every served name is not a
|
||||
machine — now the template's choice of `.Names` or `.Machines`
|
||||
- mesh-controller `internal/catalogue/roster.go` (the mechanism), `internal/overlay/generator.go`
|
||||
(the built-in `/etc/hosts` template), `internal/catalogue/manifest.go` (`RosterFile`)
|
||||
- mesh-catalog `modules/dnsmasq/module.json` (the zones template, dnsmasq's own)
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
---
|
||||
|
||||
# 121. A system seat is named for its scope, and a module may define its own
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md) made seats a closed set the
|
||||
control plane defines: a well-formed name no longer becomes a seat by being claimed, so a person can
|
||||
read what a mesh can have and who fills each role. It left two things unsettled that the growing set
|
||||
now exposes:
|
||||
|
||||
- **The names carry no rule.** `mesh-controller`, `mesh-store`, `mesh-broker` are named for the mesh;
|
||||
beside them sit `the-artifact-store`, `the-build-machine`, `the-dns-port`, `the-showcase`,
|
||||
`the-uplink` — a second naming style with no principle behind it. A reader cannot tell a seat's
|
||||
scope from its name, and the mesh's own roles do not look like the mesh's.
|
||||
- **The set is the *only* place a seat may be defined.** A module claiming any name not in the
|
||||
control plane's set is refused. That is right for *system* roles — one broker, one packet filter
|
||||
per node — but it means a module can never define a role of its own: a demo module's
|
||||
`the-showcase`, a future application's coordination role, must be smuggled into the control plane's
|
||||
set or not exist. The control plane ends up holding roles that are not the mesh's to define.
|
||||
|
||||
Reviewing the set against these also found seats whose *scope* or *membership* is wrong, not just
|
||||
their name — the review is the occasion to fix those too.
|
||||
|
||||
## Decision
|
||||
|
||||
**A system seat — one the control plane defines — is named for its scope:**
|
||||
|
||||
- **`mesh-*`** for a mesh-scoped seat: one holder in the whole mesh, a role the mesh has once
|
||||
(`mesh-controller`, `mesh-store`, `mesh-broker`, `mesh-git`, …). A `mesh-*` seat is always held by
|
||||
a module **on a named node** — `mesh-git` is gitea *on novox*, not "gitea"; another node running
|
||||
gitea does not hold `mesh-git` unless it is the holder. The seat is the mesh's single answer for
|
||||
the role, and which node answers is part of what the seat records.
|
||||
- **`node-*`** for a node-scoped seat: one holder per node, a role each machine has at most once
|
||||
(`node-packet-filter`, `node-intrusion-prevention`, `node-uplink`, …).
|
||||
|
||||
The three already-`mesh-*` seats keep their names; the rest are renamed by this rule. The scope a
|
||||
name declares must match the seat's actual scope — a `mesh-*` seat at node scope, or the reverse, is
|
||||
a contradiction the reader is entitled to trust is impossible.
|
||||
|
||||
**The control plane defines only system seats. A module may define its own.** A seat named `mesh-*`
|
||||
or `node-*` is the control plane's, and claiming one the control plane does not define is refused as
|
||||
before. Any *other* name is a **module-defined seat**: valid when the module declaring the claim also
|
||||
declares the seat (its name, scope, and — if any — the protocol its holder speaks). The control plane
|
||||
enforces one-holder-per-scope for it exactly as for its own, but does not otherwise know what it
|
||||
means. So an application can coordinate its own instances through a seat of its own, and the mesh's
|
||||
closed set stays what its name says it is: the *system's* roles, not everyone's.
|
||||
|
||||
**Specific seats this settles:**
|
||||
|
||||
- **`the-build-machine` → `mesh-build-machine`, and its scope becomes mesh.** There is one build
|
||||
machine in the mesh (the builder on novox), not one per node. Node scope said the opposite. It
|
||||
delivers no provision; it is the mesh's single build machine.
|
||||
- **`the-private-network` → `mesh-private-network`, held by the network *server* on one node.** Today
|
||||
it is node-scoped and held on every node, with a stated (untested) story that a different VPN could
|
||||
hold it per machine — which would force every provider module to independently implement receiving
|
||||
and applying the controller-composed configuration. The mesh does not work that way and should not
|
||||
pretend to: **one mesh decides one private network.** The seat is mesh-scoped, held by the server
|
||||
module (WireGuard on the hub, novox). A machine that joins is given a **client module** that
|
||||
receives the composed configuration and applies it; when a node joins, the mesh emits each node's
|
||||
configuration so all of them know each other at once. This drops per-node VPN choice deliberately —
|
||||
the private network is nox-mesh's own, and it defines the nodes' configuration rather than being
|
||||
assembled from each node's opinion. (Implementation: the overlay generator's per-node computation
|
||||
is unchanged; what changes is the seat's scope and the server/client split of the module.)
|
||||
- **`the-showcase` → removed from the set; it becomes a module-defined seat.** It is a demo module's
|
||||
own coordination role, claimed by nothing else and held nowhere. It is the first module-defined
|
||||
seat, and the reason the rule above is needed rather than hypothetical.
|
||||
- **`the-dns-port` → `node-dns-resolver`** (the daemon that binds `:53`), kept distinct from
|
||||
**`the-resolver-configuration` → `node-resolver-config`** (what writes `resolv.conf`). Two roles,
|
||||
two seats; the rename must not blur them.
|
||||
- **`the-packet-filter` → `node-packet-filter`** and **`the-intrusion-prevention` →
|
||||
`node-intrusion-prevention`** — names kept as-is but for the prefix. "Packet filter" stays distinct
|
||||
from "firewall", which would swallow intrusion-prevention too.
|
||||
- **`the-uplink` → `node-uplink`** ([ADR 0125](0117-a-machines-uplink-is-a-seat.md)). Unheld, so it
|
||||
renames with no migration.
|
||||
- **The registry seats — `the-artifact-store`, `npm-package-registry` (→ `mesh-artifact-store`,
|
||||
`mesh-npm-package-registry`) — and `git` (→ `mesh-git`) — are decided but deferred.** They each
|
||||
*deliver* a provision, so renaming them is a delivering-seat migration: a holder that stops
|
||||
resolving mid-flight takes a provision away from every consumer. That risk is not worth carrying in
|
||||
the same pass as the node-* renames, so they keep their names until done deliberately.
|
||||
|
||||
**`distribution` stays the mesh's registry; only `verdaccio` is retired.** An earlier draft of this
|
||||
record had the registry consolidating onto gitea and `distribution` retired — that was reversed:
|
||||
`distribution` is the standalone OCI registry serving every `artifact-store://…@sha256` image (the
|
||||
control plane's own included), and the mesh keeps it. `verdaccio` was a *second* npm registry;
|
||||
gitea already provides `npm-package-registry`, so verdaccio is redundant and is removed. It is only
|
||||
in the catalogue (never registered in the running mesh), so removing it is deleting the module — no
|
||||
migration, nothing to strand.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **A reader learns a seat's scope from its name.** `mesh-*` is mesh-wide and one; `node-*` is
|
||||
per-machine. The mesh's own roles finally look like the mesh's.
|
||||
- **Applications get their own seats** without the control plane learning their meaning. The closed
|
||||
set shrinks to what it should be — the system's roles — and stops being where unrelated roles hide.
|
||||
- **The renames are a coordinated migration, not a rename.** A held seat's name lives in three places
|
||||
that must move together: the control plane's set (`seats.go`), every claiming manifest, and what
|
||||
each node reports it holds (re-derived by re-registering the manifest and re-pushing). A seat
|
||||
renamed in one place and not the others stops resolving to its holder — and for a *delivering* seat
|
||||
(`mesh-store`→postgres, `mesh-broker`→amqp, the registry seats) that is a mesh-wide provision
|
||||
outage, the same failure mode as a schema change hitting an old manifest. So: the non-delivering
|
||||
`node-*` seats and `mesh-build-machine` migrate as one tested controller+catalogue change;
|
||||
`node-uplink` is free (unheld); the delivering registry seats are deferred to their own pass.
|
||||
- **The node-* migration was done as one controlled step, and it froze briefly.** Deploying the new
|
||||
controller made it reject the still-old-named claims in the stored manifests, so composition stopped
|
||||
for the affected nodes until each manifest was re-registered under its new name; running services
|
||||
were untouched, and the window was seconds. This is the coordinated-migration cost named above,
|
||||
paid once — and the reason the *delivering* registry seats, whose freeze would be a provision
|
||||
outage rather than a compose pause, are not folded into the same pass.
|
||||
- **`distribution` is not retired.** It stays as the registry; only `verdaccio` (a redundant second
|
||||
npm registry) is removed. The mesh keeps one OCI registry (`distribution`) and gitea for npm/git —
|
||||
the "one registry, on gitea" idea was considered and dropped.
|
||||
- **The private network stops pretending to be swappable per node.** The gain is a coherent
|
||||
server/client model matching how the controller already composes configuration; the cost is that
|
||||
choosing a different VPN is now a mesh-wide change, not a per-node one — accepted.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md) — the closed set this refines
|
||||
- [ADR 0125](0117-a-machines-uplink-is-a-seat.md) — `the-uplink`, renamed here to `node-uplink`
|
||||
- [ADR 0079](0079-the-foundation-seats-are-named-after-their-servers.md) — the original `mesh-*` seats
|
||||
whose naming this generalises
|
||||
- [to-be 26](../03-DESIGN/01-to-be/26-the-seats.md) — the seat table, updated by this
|
||||
- mesh-controller `internal/catalogue/seats.go` (the set and claim validation),
|
||||
`internal/overlay/generator.go` (the private network as server + client)
|
||||
@@ -0,0 +1,107 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
supersedes-in-part:
|
||||
- 0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
- 0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md
|
||||
---
|
||||
|
||||
# 122. A seat is data the controller owns, and a rename is a database update
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md) made the seats a closed set the
|
||||
control plane defines, and [ADR 0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md)
|
||||
named them by scope. Both were right about *what* a seat is. Both left it defined the wrong *way*:
|
||||
**the set is a hardcoded Go slice compiled into the controller, and everything references a seat by
|
||||
its name as a string literal.** Renaming `the-packet-filter` to `node-packet-filter` this session
|
||||
took, in one pass:
|
||||
|
||||
- an edit to the Go slice in `internal/catalogue/seats.go`, recompiled into a new controller image;
|
||||
- an edit to a `const gitSeat = "git"` in *production* control-plane code (`source.go`), because a
|
||||
seat's name was hardcoded where a repository's home is resolved;
|
||||
- edits to every claiming manifest in the catalogue, each re-registered;
|
||||
- a controller **rebuild and redeploy**, which — because the running controller then refused the
|
||||
still-old-named claims in stored manifests — **froze composition** for the affected nodes until
|
||||
each manifest was re-registered under its new name;
|
||||
- the same coupling in the **build machine**, which embeds the same seat set and refused to build
|
||||
anything claiming a name it did not yet know;
|
||||
- a **deadlock** when the build machine's own seat was renamed, since the old builder could not
|
||||
build the new builder whose manifest claimed a name it rejected.
|
||||
|
||||
None of that is what a rename should cost. A rename is the operator changing a label. It should be a
|
||||
single write, and nothing should have to be rebuilt, refused, or unfrozen. The set being *closed*
|
||||
(0110) and *named by scope* (0121) are good rules; **the set being code is the mistake.** When
|
||||
adhering to the design means twenty steps and a `const` in the resolver, the design is what to fix.
|
||||
|
||||
## Decision
|
||||
|
||||
**The seat set is data the control plane owns, not code it is compiled from.** The seats live in a
|
||||
table in the controller's store — one row per seat: a **stable id**, a `name`, a `scope`, what it
|
||||
`delivers` (a provision, or nothing), and the record that decided it. The rows are seeded by a
|
||||
migration (the closed set 0110 defines still ships with the mesh), and thereafter they are ordinary
|
||||
data the control plane reads and writes.
|
||||
|
||||
**A seat is referenced by its stable id, never by its name.** A claim, a held-seat record, and any
|
||||
control-plane code that must name a seat (the git-seat resolver, the artifact-store guard) hold the
|
||||
**id**. The `name` is a label for people and for what a manifest writes; it is resolved to an id
|
||||
once, when a claim is registered. So:
|
||||
|
||||
- **A rename is one `UPDATE seats set name = … where id = …`.** Nothing is recompiled, nothing is
|
||||
re-registered, nothing is refused, nothing freezes. Held records and claims already point at the
|
||||
id, so they follow the rename for free. The build machine is not involved, because the build
|
||||
machine validates a claim against the set it reads from the mesh, not one baked into its image.
|
||||
- **Adding or removing a seat is an `INSERT`/`DELETE`** (within the closed-set discipline: a change
|
||||
to the set is still a decision with a record — the record is now a row's `decided` column and an
|
||||
ADR, not a line of Go). No controller release is needed to change the roster of roles.
|
||||
- **Production code stops hardcoding names.** `const gitSeat = "git"` becomes a lookup of the seat
|
||||
that delivers the `git` provision (or a well-known id), so renaming its label cannot break the
|
||||
code that finds a repository's forge.
|
||||
|
||||
**What does not change** (0110 and 0121 still hold): a seat is still a module assignment from a
|
||||
closed set; there is still one holder per scope; a delivering seat is still the single answer for
|
||||
its provision; system seats are still `mesh-*`/`node-*` and a module may still define its own. Only
|
||||
their *storage and reference* change — from a compiled slice keyed by name to a table keyed by id.
|
||||
|
||||
**A manifest still claims by name, and that is fine.** A manifest is written by a person and names
|
||||
the seat in words; the mesh resolves the name to an id at registration and stores the id. If a
|
||||
seat's name changes, manifests written against the old name are updated in the catalogue like any
|
||||
other edit (and the mesh can keep the old name as an alias row during a transition so nothing breaks
|
||||
in the window) — but the *control plane* never has to change or redeploy for it, which is the whole
|
||||
point. The heavy, mesh-wide, freeze-prone half of a rename disappears; only the ordinary catalogue
|
||||
edit remains.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **A rename, and a set change, become operations, not releases.** The pain this session paid —
|
||||
three freezes, a builder deadlock, hand-resolved manifests — is designed out. The seat migrations
|
||||
still outstanding (the delivering registry seats, and the private network's scope change) should
|
||||
wait for this: done as data, each is a write, not a coupled multi-repo deploy.
|
||||
- **The controller gains a small table and a seed migration**, and its seat lookups change from
|
||||
slice scans to id-keyed reads. `SeatNamed`, `SeatDelivering`, `claimProblems` read the table.
|
||||
- **The build machine reads the set from the mesh** (it already talks to the control plane), rather
|
||||
than embedding it — which removes the controller/builder seat coupling that made every breaking
|
||||
seat change a two-sided deadlock (see [to-be 30](../03-DESIGN/01-to-be/30-the-mesh-updates-itself-on-a-push.md)).
|
||||
- **The closed set is still closed.** Data being editable is not the set being open: changing it is
|
||||
still a decision, still recorded. What changes is that recording it no longer means shipping a
|
||||
binary.
|
||||
- **This is a real refactor**, touching the store schema, the seat lookups, claim registration
|
||||
(name→id resolution), and the held-seat records. It is worth its own build; until it lands, the
|
||||
current compiled set stands and further renames are held rather than forced through the heavy path.
|
||||
- **Config on a seat is still the module's** (the question that surfaced this): a seat row carries
|
||||
the seat's own metadata (scope, delivers, protocol), not a module's configuration — that stays in
|
||||
the holding module's manifest ([ADR 0046](0046-a-module-configuration-is-its-assignments-not-its-manifest.md)). Making
|
||||
seats data does not make them a config store.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md),
|
||||
[ADR 0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md) — the seat
|
||||
rules this keeps, whose *storage* it changes
|
||||
- [to-be 30](../03-DESIGN/01-to-be/30-the-mesh-updates-itself-on-a-push.md) — the controller/builder
|
||||
seat coupling and the breaking-change freeze this removes for seat changes
|
||||
- mesh-controller `internal/catalogue/seats.go` (the compiled slice this replaces),
|
||||
`cmd/mesh-controller/source.go` (`const gitSeat`, the hardcoded name this removes)
|
||||
@@ -0,0 +1,133 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: superseded
|
||||
superseded-by: 02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md
|
||||
date: 2026-09-26
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
---
|
||||
|
||||
# 125. The bus is the only broker
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0106](0106-the-bus-is-nats.md) moved the mesh's bus to NATS and kept the AMQP broker "as a
|
||||
module with one purpose — the predecessor's clients", retiring with the last of them.
|
||||
[Design 25](../03-DESIGN/01-to-be/25-the-bus-on-nats.md) repeats that: a compatibility module with
|
||||
a single purpose and a retirement condition.
|
||||
|
||||
**It is not single-purpose, and was not when that was written.** Two modules of the *new* mesh
|
||||
declare `requires: ["amqp"]` and are answered by the broker module's own provisioner:
|
||||
|
||||
- `amqp-ping`, whose source says it "exists to PROVE the grant end to end: the mesh gave it a
|
||||
scoped login and a vhost of that name on the lavinmq provider";
|
||||
- `amqp-email-forwarder`, which uses it for work.
|
||||
|
||||
What that provisioner answers is **not the mesh's bus**. Its own comment draws the line: a
|
||||
consumer gets "its own message broker, isolated from every other consumer's by the vhost
|
||||
boundary… a broker of its own, not a shared account on the mesh's control-plane broker" —
|
||||
vhost-per-login, "the exact analog of postgres's database-per-login."
|
||||
|
||||
So two different things wear the word *broker*: the mesh's nervous system, and a private message
|
||||
broker handed to a module as a resource, the way a database is. The first is being replaced. The
|
||||
second was never examined, and on the retirement condition ADR 0106 sets, it disappears with no
|
||||
successor and nothing notices — a module of the new mesh left requiring something no provider
|
||||
answers.
|
||||
|
||||
The operator's direction, asked at the point this surfaced: **NATS is the heart of the
|
||||
application** — not a component it contains, and not a thing to reproduce the predecessor's
|
||||
shapes on.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Carry the private broker forward onto NATS** — each requiring module gets its own NATS
|
||||
account, provisioned like a database. Rejected on three counts. It reproduces the
|
||||
predecessor's shape on the new bus, which is the thing this whole move exists to stop. It
|
||||
gives the mesh two messaging models, so "how does a module send a message" has two answers
|
||||
depending on a manifest line. And NATS accounts isolate subject spaces *entirely*: a module
|
||||
inside its own account cannot reach the mesh's bus at all, so it would hold two connections
|
||||
and two identities to do one job.
|
||||
2. **Keep the compatibility broker indefinitely** for the mesh's own modules. Rejected: its
|
||||
retirement condition is the point of it. A module of the new mesh depending on the retired one
|
||||
keeps the predecessor alive permanently, which is the opposite of a compatibility module.
|
||||
3. **One bus. A module's messaging is subjects on it, scoped by what it declares.** Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**The bus is the only broker.** NATS is the mesh's one messaging system, and every module's
|
||||
messaging is subjects on that bus under its own account, scoped by its `emits` and `consumes`
|
||||
([ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md)). There is no second
|
||||
broker, and none is handed to a module as a resource.
|
||||
|
||||
**The `amqp` interface is not carried forward.** It leaves the set of things a module may require
|
||||
and retires with the compatibility broker rather than gaining a successor.
|
||||
|
||||
Concretely, in the controller's seat table: **the `mesh-broker` seat delivers nothing.** It
|
||||
currently reads `Delivers: "amqp"` — the seat's holder answers a requirement for a broker — and
|
||||
under this decision it joins `mesh-controller` and `the-catalogue`, the foundation seats that
|
||||
deliver no provision at all. The bus is not something a module asks for; it is what a module is
|
||||
reached through.
|
||||
|
||||
- `amqp-email-forwarder` moves to the bus like any module: what it emits and consumes, declared,
|
||||
and the account follows.
|
||||
- `amqp-ping`'s *purpose* is kept and its mechanism is not. Proving end to end that a module
|
||||
receives scoped messaging it did not configure itself is worth a probe; it becomes a probe of
|
||||
the bus, and its assertion changes from "I reached my own vhost" to "I reached exactly my
|
||||
subjects and was refused the rest."
|
||||
|
||||
**A module that wants a queue of its own has one already**: a subject nothing else may publish to
|
||||
and a durable consumer of its own, both derived from its declaration. What it does not get is a
|
||||
server of its own.
|
||||
|
||||
**The mesh's own streams are the controller's, created at genesis, not provisioned** — and
|
||||
`EVENTS` is one stream, closing the question [design 25](../03-DESIGN/01-to-be/25-the-bus-on-nats.md)
|
||||
§11 left open. The reason is not preference but **bootstrapping**: a provisioner is a module, and
|
||||
a module needs a bus account before it can run at all. Anything the bus itself is made of must
|
||||
exist before the first module starts, so it is composed as configuration
|
||||
([ADR 0106](0106-the-bus-is-nats.md): never through a management API) rather than provisioned by
|
||||
something that could not yet be running.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Design 25 gains the distinction and loses the "single purpose" claim**; its §11 question about
|
||||
the `EVENTS` stream closes here.
|
||||
- **Nothing in [design 27](../03-DESIGN/01-to-be/27-a-module-requires-the-mesh-resolves.md)'s
|
||||
model changes** — the four provider kinds, the contract, resolution all stand, and it never
|
||||
enumerated interfaces, so there is nothing to strike from it. What changes is that messaging
|
||||
leaves the set of things resolved at all: every module has it by existing.
|
||||
- **One line of the controller's seat table changes**, and it is the load-bearing one:
|
||||
`mesh-broker` stops declaring what it delivers. A requirement for `amqp` then resolves to
|
||||
nothing and is refused at assignment, which is how the two modules below are found rather than
|
||||
discovered at runtime.
|
||||
- **Two modules have conversion work**, and it belongs to step 4 of
|
||||
[ADR 0116](0116-the-bus-is-built-in-five-steps.md), with the flows. Neither blocks step 1.
|
||||
- **The compatibility broker becomes what ADR 0106 already called it** — single-purpose — once
|
||||
those two have moved. That record's claim was wrong when written and is made true by this one.
|
||||
- **What got harder:** a module that genuinely wanted an isolated server — a tenant boundary at
|
||||
the broker rather than at the subject — no longer has that option, and would have to argue for
|
||||
it as a new decision. That is the intended cost: one bus is the point.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A module's messaging works with no `requires` line for it.** A lab bed: a module declaring
|
||||
only `emits` and `consumes` reaches its subjects, and is refused every other — which is
|
||||
[ADR 0116](0116-the-bus-is-built-in-five-steps.md) step 1's permission bed, already required.
|
||||
- **Nothing requires `amqp`.** With the seat delivering nothing, a module still declaring it is
|
||||
refused at resolution — the existing "requirement no provider answers" path, not a new check. A
|
||||
catalogue test asserts no module declares it once the two have moved.
|
||||
- **The probe proves the claim it is named for.** `amqp-ping`'s successor fails if a module can
|
||||
reach a subject outside its declaration, not merely if it cannot reach its own.
|
||||
- **The compatibility broker's retirement condition can actually be met.** A check that no module
|
||||
of the mesh — as opposed to a predecessor client — holds a connection to it.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0106](0106-the-bus-is-nats.md) — the bus is NATS; corrected here on what the compatibility
|
||||
broker serves.
|
||||
- [ADR 0116](0116-the-bus-is-built-in-five-steps.md) — the steps; the conversions land in step 4.
|
||||
- [ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md) — the scoping that
|
||||
makes one bus safe.
|
||||
- [design 25](../03-DESIGN/01-to-be/25-the-bus-on-nats.md),
|
||||
[design 27](../03-DESIGN/01-to-be/27-a-module-requires-the-mesh-resolves.md) — the two documents
|
||||
this changes.
|
||||
@@ -0,0 +1,145 @@
|
||||
---
|
||||
topic: the tiers
|
||||
status: accepted
|
||||
date: 2026-09-26
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
---
|
||||
|
||||
# 126. A module declares its own seats; the mesh reserves its own
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md) closed the set of seats. Its
|
||||
evidence was strong and still is: nothing could answer *which seats does this mesh have, and who
|
||||
holds each*. Answering it meant reading every manifest in two repositories and then the
|
||||
controller's own code, and when that enumeration was done by hand while writing the record, **it
|
||||
reported eleven claims where there were thirteen.** The fix was a table in the controller, and
|
||||
adding a seat became a decision.
|
||||
|
||||
What that table cannot express is the architecture [ADR 0125](0125-the-bus-is-the-only-broker.md)
|
||||
opened. With one bus and no private brokers, a module offering a service to other modules offers
|
||||
it as **a role on the bus**: a set of subjects, exactly one holder, addressed by what it does
|
||||
rather than by which module or node provides it. A telegram sender, a licensing master, anything
|
||||
a mesh might want one of. Under a closed table, adding any of those means editing the controller
|
||||
— so a capability contributed by a module would require a change to the mesh itself, which is the
|
||||
coupling the module system exists to prevent.
|
||||
|
||||
**The two requirements look opposed and are not.** 0110 needs the set *enumerable*. The
|
||||
architecture needs it *extensible*. Those conflict only if enumerable means *written down in one
|
||||
place by hand* — which is exactly the property that let the count drift in the first place.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Keep the closed table, add each new seat by decision.** Rejected. Every capability a module
|
||||
contributes would need a change to the controller and a record before it could be offered, and
|
||||
the mesh would carry the names of services it does not itself implement.
|
||||
2. **Free-form seats, as before 0110.** Rejected for 0110's own reason, unchanged: nothing can
|
||||
say what a mesh has, and a name invented at a claim site is a name nobody can explain later.
|
||||
3. **A set that is closed at any moment and derived rather than maintained**, with the mesh's own
|
||||
seats reserved by name. Adopted. 0110 weighed options 1 and 2 and never considered this one.
|
||||
|
||||
## Decision
|
||||
|
||||
**A seat may be declared by a module, and the set of seats a mesh has is derived: the mesh's own,
|
||||
plus those declared by every module it has registered.** The set is still closed — a seat named
|
||||
nowhere is refused — but it is computed from the catalogue rather than written in the controller.
|
||||
|
||||
Everything 0110 decided about what a seat *is* stands untouched: one holder at its scope; a
|
||||
definition says which seats a module *can* hold and an assignment says which it *does*; holding
|
||||
one may deliver a provision; a seat makes a role singular, never a module.
|
||||
|
||||
**Enumeration is a query, not an inventory.** The catalogue knows every registered manifest, so
|
||||
"which seats does this mesh have, and who holds each" is answered by asking it. This is a
|
||||
stronger answer than the table gave, not a weaker one: a derived list cannot drift from reality,
|
||||
and drift is how the hand-made count came out at eleven of thirteen.
|
||||
|
||||
**The mesh's own seats are reserved by prefix.** Every seat the mesh itself defines is named
|
||||
`mesh-*`, and a module declaring any `mesh-*` name is refused at registration. The prefix *is*
|
||||
the reservation rule — no list of reserved names to maintain, and no way for the mesh's own
|
||||
namespace to be colonised by a manifest. This requires renaming the seats that drifted from
|
||||
[ADR 0079](0079-the-foundation-seats-are-named-after-their-servers.md)'s convention: `the-catalogue`
|
||||
becomes `mesh-catalog`, `git` becomes `mesh-git`, and the node-scoped `the-build-machine`,
|
||||
`the-dns-port`, `the-intrusion-prevention`, `the-packet-filter`, `the-private-network`,
|
||||
`the-resolver-configuration`, `the-showcase` take the same prefix.
|
||||
|
||||
The mesh's seats stay the mesh's for a reason that does not apply to a module's: **the mesh's own
|
||||
code looks them up by name.** The resolver *is* the thing that finds the store. `mesh-store` is
|
||||
not a convention the controller follows, it is an identifier the controller dereferences.
|
||||
|
||||
**A declared seat carries a protocol.** A module declaring a seat says what may be sent to it,
|
||||
what it emits, and what it serves. The holder must satisfy it; a module may not claim a seat whose
|
||||
protocol it does not implement. Callers declare that they use the *seat*, never the module, so
|
||||
replacing the implementation changes nothing for any caller.
|
||||
|
||||
**A seat is for a role; an event stays addressed to its emitter.** The two are not
|
||||
interchangeable and the choice is not stylistic. An event is *this happened to me* — the emitter's
|
||||
identity is the meaning, which is why the envelope carries source, node and time
|
||||
([ADR 0042](0042-the-shape-of-an-event-on-the-wire.md)); routing it through a role would erase the
|
||||
provenance an audit needs. A seat is *this capability, whoever provides it* — where not knowing
|
||||
the holder is the point. Publish an event when the fact is about you; declare a seat when you are
|
||||
offering something another module could offer instead.
|
||||
|
||||
**Two modules declaring the same seat name is refused at registration**, second one loses.
|
||||
Registration is the last moment the mesh can still say no, and a seat name meaning two different
|
||||
protocols is the failure nobody could diagnose afterwards.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **The controller's seat table stops being the set** and becomes the mesh's own reserved entries.
|
||||
Resolution reads the catalogue for the rest.
|
||||
- **Ten seats are renamed.** A rename is a migration, not an edit: existing assignments hold the
|
||||
old names, so the change carries a mapping and is applied once, and the lab beds that name seats
|
||||
are updated with it.
|
||||
- **A `uses` naming an undeclared seat is refused at registration**, which is where 0110's
|
||||
guarantee lands under this model — the same refusal, at the same moment, from a derived set.
|
||||
- **Adding a capability stops requiring a decision record.** That is a real loss of governance and
|
||||
the intended trade: the argument for a seat's existence moves into the module that declares it,
|
||||
where it is reviewed as part of the manifest. The mesh's own seats keep the old bar.
|
||||
- **[ADR 0041](0041-events-are-a-relationship.md)'s machinery claim is already stale** for a
|
||||
different reason, and is corrected in place there under the rule in
|
||||
[`README.md`](README.md) — a progressive insight: on JetStream a subscription is a durable
|
||||
consumer, a real object someone must create.
|
||||
- **What got harder:** a seat's protocol is now a compatibility surface between modules that do
|
||||
not know each other. Changing one breaks callers already bound to it, and nothing here says how
|
||||
that is versioned. It is the first thing to answer in the design, and the thing most likely to
|
||||
hurt later rather than now.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **The overview answers, and is right.** A command lists every seat, its scope, its protocol and
|
||||
its holder, derived from the catalogue — and a test asserts the count against a fixture mesh,
|
||||
because an enumeration nobody checks is how thirteen became eleven.
|
||||
- **`mesh-*` is refused to a module.** A registration test: a manifest declaring `mesh-anything`
|
||||
is refused, naming the prefix as the reason.
|
||||
- **An undeclared seat is refused.** A registration test on `uses`, and a resolution test that
|
||||
nothing reaches runtime unresolved.
|
||||
- **A second declarer loses.** A registration test: two manifests, same seat name, the second
|
||||
refused and the first untouched.
|
||||
- **A holder must satisfy the protocol.** A claim whose module does not serve what the seat
|
||||
declares is refused at assignment, not discovered when a caller times out.
|
||||
|
||||
## Progressive insight
|
||||
|
||||
> **Progressive insight — 2026-09-26.** *A seat rename is not a data migration.* This record's
|
||||
> consequences say "a rename is a migration, not an edit: existing assignments hold the old
|
||||
> names, so the change carries a mapping and is applied once". Implementing it showed there is
|
||||
> nothing stored to migrate: a seat's holding is **derived at resolution** from the claims in
|
||||
> manifests (`resolve.go` builds it each time), never written down, so no recorded name is left
|
||||
> pointing at the old one. What exists is source — the controller's seat table, the manifests
|
||||
> that claim them, and a manifest that may be registered later from its own repository. So the
|
||||
> change is an edit plus a **kept** rename table, which tells a manifest written against an old
|
||||
> name what it became rather than refusing it as unknown.
|
||||
>
|
||||
> The decision — that modules declare seats, that the mesh reserves `mesh-*`, and that the ten
|
||||
> are renamed — is unchanged. Only the shape of the work was wrong.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0110](0110-a-seat-is-a-module-assignment-from-a-closed-set.md) — superseded here; its
|
||||
requirement is kept and only its mechanism replaced.
|
||||
- [ADR 0125](0125-the-bus-is-the-only-broker.md) — one bus, which is what makes a role addressable.
|
||||
- [ADR 0079](0079-the-foundation-seats-are-named-after-their-servers.md) — the naming convention
|
||||
the reserved prefix restores.
|
||||
- [ADR 0041](0041-events-are-a-relationship.md) — the event half of the boundary drawn here.
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: superseded
|
||||
superseded-by: 0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
date: 2026-09-26
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
supersedes: 02-DECISIONS/0125-the-bus-is-the-only-broker.md
|
||||
---
|
||||
|
||||
# 127. AMQP is a provision, not the bus
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0125](0125-the-bus-is-the-only-broker.md) decided that the bus is the only broker, and went
|
||||
one step further than it had grounds for: it also decided that the `amqp` **interface** — a module
|
||||
requiring a message broker of its own — "is not carried forward" and "retires with the
|
||||
compatibility broker rather than gaining a successor", with the two modules declaring it converted
|
||||
to the bus in step 4.
|
||||
|
||||
The operator's correction: **AMQP is deprecated as the mesh's transport, not abolished as a
|
||||
service.** The broker module keeps running and keeps answering `amqp` requirements. It is no
|
||||
longer a core part of the mesh — *"it's just a module like mssql now."*
|
||||
|
||||
**What 0117 conflated** is two different reasons a module might ask for a broker, which look
|
||||
identical in a manifest:
|
||||
|
||||
1. **To talk to other modules.** Wrong under one bus, and the thing 0117 was right to refuse: a
|
||||
private broker used as inter-module transport is a second bus, with every guarantee crossing a
|
||||
seam and no scoping the mesh can see.
|
||||
2. **Because it genuinely needs an AMQP broker**, the way something needs a database — a queue for
|
||||
its own internals, or interop with software that speaks AMQP and nothing else. That is a
|
||||
backing service, and the mesh has a word for backing services already.
|
||||
|
||||
0117 saw the first and legislated against both. The second is ordinary, and forbidding it would
|
||||
make the mesh unable to run a large class of perfectly normal software while claiming that as
|
||||
architecture.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Keep 0117 as written** — retire the interface, convert the two modules. Rejected by the
|
||||
operator, and wrongly reasoned besides: it treats "needs an AMQP broker" as always a mistake.
|
||||
2. **Keep the broker as the predecessor's compatibility module**, as ADR 0106 framed it, with a
|
||||
retirement condition. Rejected: it is not single-purpose and its clients are not only the
|
||||
predecessor's, so the retirement condition describes a day that will not come.
|
||||
3. **The broker is an ordinary provider module of an ordinary provision.** Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**The mesh's bus is NATS and only NATS.** Everything 0117 decided about *the bus* stands: one bus,
|
||||
a module's messaging is subjects on it scoped by what it declares, no module is handed a bus of
|
||||
its own, and the `mesh-broker` seat is the NATS server's.
|
||||
|
||||
**`amqp` remains a provision a module may require**, answered by the broker module the way
|
||||
`postgres-database` is answered by the store module or a database is answered by mssql. It is not
|
||||
deprecated as an interface; the software behind it is simply no longer the mesh's nervous system.
|
||||
|
||||
**The broker module stops being foundation.** It claims no seat — `mesh-broker` is the NATS
|
||||
server's — it is not raised at genesis, nothing in the mesh requires it, and a mesh that never
|
||||
installs it is a complete mesh. It is installed when something wants it, like any other provider.
|
||||
|
||||
**The rule that survives, stated so it can be applied:** *inter-module communication goes over the
|
||||
bus.* A module may hold a broker, a database or a cache as a backing service; it may not use one
|
||||
as a channel to another module. The line is not which software is involved, it is whether a second
|
||||
module is on the other end.
|
||||
|
||||
**Neither `amqp-ping` nor `amqp-email-forwarder` needs converting.** 0117 put that work in step 4;
|
||||
it is removed. They require a backing service and a provider answers.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **The "compatibility broker" framing is wrong and goes.** There is no `lavinmq-compat`, no
|
||||
single purpose and no retirement condition. Design 25 §5 is corrected.
|
||||
- **[ADR 0106](0106-the-bus-is-nats.md)'s progressive insight was itself wrong** and is corrected
|
||||
by a second one there. It said 0117 would make 0106's "one purpose — the predecessor's clients"
|
||||
sentence true by moving the mesh's modules off. Nothing moves off; the sentence is simply not
|
||||
what the broker is.
|
||||
- **The seat change stands**, for a better reason than 0117 gave: not because a broker cannot be
|
||||
provisioned, but because *this* broker is not the mesh's bus. The broker module drops its
|
||||
`mesh-broker` claim and the `nats` module takes it.
|
||||
- **Step 4 loses two conversions**; step 1 and the WBS are otherwise unaffected.
|
||||
- **What got harder:** the rule is now a judgement rather than a prohibition. "Is this a backing
|
||||
service or a channel to another module?" has to be asked in review, where 0117 could have
|
||||
answered it with a parser. That is the honest cost of allowing the legitimate case.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A module's own messaging needs no `requires`.** The check from 0117, unchanged: a module
|
||||
declaring only `emits` and `consumes` reaches its subjects and is refused every other.
|
||||
- **The broker holds no seat.** A manifest test: the broker module claims nothing, and a mesh
|
||||
raised without it is complete — genesis names it nowhere.
|
||||
- **`amqp` resolves like any provision.** A resolution test: a module requiring it is answered by
|
||||
the provider, refused when none is assigned, and neither case touches the bus.
|
||||
- **What cannot be checked mechanically**, and is said rather than implied: that a module holding
|
||||
a broker is not using it to reach another module. Review, not a parser.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0125](0125-the-bus-is-the-only-broker.md) — superseded; its ruling on the bus is kept
|
||||
whole and only its ruling on the interface is reversed.
|
||||
- [ADR 0106](0106-the-bus-is-nats.md) — the bus is NATS; its compatibility-broker framing is
|
||||
corrected here.
|
||||
- [ADR 0126](0126-a-module-declares-its-own-seats.md) — seats, including the one the NATS server
|
||||
now holds alone.
|
||||
@@ -0,0 +1,123 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-26
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
---
|
||||
|
||||
# 128. The mesh bus is required, not ambient
|
||||
|
||||
|
||||
> **Pointer repointed, 2026-09-27.** This record was written extending
|
||||
> [ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)), which
|
||||
> [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md) has since superseded — AMQP is
|
||||
> not a provision at all. Nothing decided here changes; the frontmatter now rests on the live record,
|
||||
> and the citations below are read with that in mind.
|
||||
|
||||
## Context
|
||||
|
||||
[Design 29](../03-DESIGN/01-to-be/32-what-a-module-declares.md) opened by saying the bus is
|
||||
*ambient*: "No module requires it, the way no module requires a filesystem. Every module gets a
|
||||
connection and an identity whether it asks or not."
|
||||
|
||||
**Two counts say that is wrong.** Of the 72 modules in the catalogue, **49 declare an own-secret
|
||||
named `broker` and 23 do not.** So the bus is not universal — nearly a third of the catalogue
|
||||
never speaks to it — and an ambient connection would mint an account, a password and a permission
|
||||
set for every one of those 23, each a credential nothing uses and everything must rotate.
|
||||
|
||||
And the 49 that do take one **each hand-write the path it lands at**
|
||||
(`own-secrets: { broker: "/var/lib/<module>/broker" }`). That is a special case doing badly what
|
||||
provisioning already does well: a consumer names where a credential lands, the mesh seals it
|
||||
there, and rotation and removal follow the same path as every other credential.
|
||||
|
||||
**The argument that made the bus ambient was narrower than it looked.**
|
||||
[ADR 0125](0125-the-bus-is-the-only-broker.md) reasoned that bus accounts cannot be provisioned
|
||||
because a provisioner is itself a module that needs an account before it can run. That is true of
|
||||
a **provisioner process**, and it is not true of a provision: the mesh's bus accounts are composed
|
||||
by the *controller*, into configuration, and the controller is not waiting on a bus account to
|
||||
exist. The circularity is real for one mechanism and absent for the other, and the earlier record
|
||||
applied it to both.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Keep the bus ambient.** Rejected on the counts above: it over-grants to 23 modules and keeps
|
||||
a hand-written path in 49.
|
||||
2. **Derive the requirement** from whether a module declares any `emits`, `consumes`, `serves` or
|
||||
`uses`. Rejected: it is the ambient model with extra inference. A reader of a manifest still
|
||||
cannot see that the module holds a bus credential, and the rule would have to be re-derived
|
||||
every time the set of bus-facing declarations grew.
|
||||
3. **The mesh bus is a provision a module requires**, delivered by the seat that holds it.
|
||||
Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**A module that speaks to the mesh requires `mesh-bus`, and receives what it needs to connect.**
|
||||
The contract is an address, a credential sealed to the module, and the trust to verify the
|
||||
server. It lands where the module's manifest says, like any provision. A module that does not
|
||||
require it gets no account, no password and no permissions — and 23 modules in the catalogue
|
||||
should get none.
|
||||
|
||||
**The `mesh-broker` seat delivers `mesh-bus`.** Its holder is the mesh's own bus, and what
|
||||
holding it delivers is the connection to that bus — which is what a seat delivering a provision
|
||||
has always meant ([design 26](../03-DESIGN/01-to-be/26-the-seats.md)).
|
||||
|
||||
**The requirement delivers the connection; the declarations shape the authority.** They are two
|
||||
different things and both stay explicit. `requires: mesh-bus` says *this module talks to the
|
||||
mesh*; `emits`, `consumes`, `serves`, `uses` and a declared seat say *what it may say and hear*,
|
||||
and the permission set is derived from those and nothing else
|
||||
([ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md)). Requiring the bus
|
||||
grants no subject; declaring a subject without requiring the bus is refused at registration as
|
||||
incoherent.
|
||||
|
||||
**The `mesh-bus` provision is answered by the controller, not by a provisioner.** This is the
|
||||
surviving kernel of ADR 0125's bootstrap argument, narrowed to what it actually supports: the
|
||||
bus's accounts are configuration the controller composes and the server reloads
|
||||
([ADR 0106](0106-the-bus-is-nats.md) — never through a management API), so there is no provisioner
|
||||
process in the path and nothing waiting on a bus account to create bus accounts. It is a provision
|
||||
whose provider is the mesh itself.
|
||||
|
||||
**A module may also provide a NATS server of its own, and that is a different interface.** Exactly
|
||||
as the AMQP broker provides `amqp` ([ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md))), a module
|
||||
may run its own NATS and offer it as a backing service. That interface is **`nats`**; the mesh's
|
||||
own bus is **`mesh-bus`**; the two are never the same name, because a manifest that said `nats`
|
||||
could mean either and the difference is the whole architecture. The rule from 0119 decides which
|
||||
is legitimate: a private bus is a backing service, never a channel to another module.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Design 29's opening is reversed.** The bus is not ambient; it is required, and the document's
|
||||
first paragraph says the opposite of this.
|
||||
- **The seat's `Delivers` is `mesh-bus`** — corrected twice in one day, which is worth recording
|
||||
rather than tidying: it read `amqp`, which was the old broker's interface; ADR 0125 emptied it,
|
||||
on the reasoning that a bus cannot be provisioned; and it is neither. The seat delivers the
|
||||
mesh's bus.
|
||||
- **`own-secrets: { broker: ... }` is retired** in favour of the provision's own delivery, across
|
||||
49 manifests. That is a mechanical change, and it belongs with the conversions in step 4 rather
|
||||
than step 1.
|
||||
- **23 modules lose a credential they never used.** Not a regression — an over-grant removed, and
|
||||
the smallest honest statement of what this buys.
|
||||
- **What got harder:** one more line in most manifests. The trade is that the line is true, and
|
||||
its absence is also true.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A module with no `requires: mesh-bus` has no account.** A composition test: the derived user
|
||||
list contains exactly the modules that require it, and the 23 that do not appear nowhere in it.
|
||||
- **Declaring a subject without requiring the bus is refused.** A registration test on a manifest
|
||||
with `emits` and no requirement, naming the contradiction.
|
||||
- **Requiring the bus grants no subject on its own.** A composition test: a module that requires
|
||||
`mesh-bus` and declares nothing else gets a connection and an empty permission set.
|
||||
- **`nats` and `mesh-bus` are distinct interfaces.** A resolution test: a module requiring `nats`
|
||||
is answered by a module providing it, never by the seat holder, and vice versa.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0125](0125-the-bus-is-the-only-broker.md) — superseded by 0119; its bootstrap argument is
|
||||
narrowed here to the case it supports.
|
||||
- [ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)) — a broker as a backing service; this
|
||||
applies the same shape to the mesh's own bus and separates the two names.
|
||||
- [ADR 0043](0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md) — authority from
|
||||
declarations, which this leaves untouched.
|
||||
- [design 26](../03-DESIGN/01-to-be/26-the-seats.md) — a seat delivering a provision.
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0122-a-seat-is-data-a-rename-is-a-database-update.md
|
||||
---
|
||||
|
||||
# 129. A seat carries the protocol of its role
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0126](0126-a-module-declares-its-own-seats.md) let a module declare a seat with its protocol:
|
||||
what work the role accepts, what it emits, what it serves. A module's own seats work that way today.
|
||||
**The mesh's own seats — the `mesh-*` set — carry no protocol at all**, only a name, a scope and the
|
||||
provision they deliver. They say who does a job and nothing about what may be said to them or by
|
||||
them.
|
||||
|
||||
That gap surfaced three times in one day, each time as a different-looking problem.
|
||||
|
||||
**A build machine.** On the bus the mesh runs on today a builder has its own account kind, created by
|
||||
its own command, with permissions written by hand: read the build queue, write to two exchanges. One
|
||||
publish to a shared exchange reached all three audiences a finished build has — whoever asked, the
|
||||
controller that records it, and the catalogue that places it in the module graph. On a bus where
|
||||
permissions are per subject those are three separate grants, and nothing derives them, because a
|
||||
builder is not a module and holds a seat that promises nothing.
|
||||
|
||||
**An event about a role rather than about a module.** The module holding the artifact-store seat
|
||||
declared an event named after a *different* module
|
||||
([issue 127](../04-ISSUES/127-a-module-event-derives-a-subject-nothing-publishes/00-report.md)). The
|
||||
bus refuses that, because a namespace belongs to who it is named for. The event is genuinely about the
|
||||
role — "the artifact store accepted an image" — and a consumer written against whichever module holds
|
||||
that role today breaks when the holder changes. There was nowhere else to put it.
|
||||
|
||||
**A catalogue catching up.** The controller answers a request for builds it may have missed by
|
||||
re-publishing them under its own name, which no consumer of the builder's subject hears. Publishing
|
||||
them under the builder's name would be the controller signing an event as another module. Answering
|
||||
into the asker's inbox needs a grant over every inbox in the mesh, which
|
||||
[design 25](../03-DESIGN/01-to-be/25-the-bus-on-nats.md) §4 refuses.
|
||||
|
||||
Three symptoms, one cause: **the mesh has roles it cannot describe.**
|
||||
|
||||
## Decision
|
||||
|
||||
**A seat carries the protocol of its role, whether the seat is a module's or the mesh's own.** The
|
||||
`mesh-*` set gains the same three fields a declared seat has — what it accepts, what it emits, what it
|
||||
serves — and the holder's authority, its work queue and its consumers are derived from them by the
|
||||
machinery that already does this for a module's seats.
|
||||
|
||||
**Builds become work submitted to a role.** The build machine seat accepts a build and emits an
|
||||
outcome. The dedicated `mesh.build.*` branch and the stream behind it retire: a work queue shared by
|
||||
several build machines is exactly what a seat's `accepts` already is, and keeping a second mechanism
|
||||
for it means two things to reason about and two places for a permission to be wrong.
|
||||
|
||||
**One publish still reaches three audiences, and now the mesh derived the subject.** A build's outcome
|
||||
is the seat's own event. Whoever asked matches it by the id their request carried; the controller
|
||||
records it; the catalogue places it. That is the fan-out the shared exchange gave for free, expressed
|
||||
as a subject rather than as a topology, and it means no holder needs permission to publish into
|
||||
anybody's inbox.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
**A dedicated principal kind for a builder**, mirroring the account the old bus issues it. Smaller: one
|
||||
addition to the composer, no change to seats, and it matches how a builder is treated today. Not taken
|
||||
because it answers one of the three symptoms and leaves the other two, and because "the builder is
|
||||
special" is a claim nobody could justify from the design — a build machine is a role the mesh has, and
|
||||
the mesh has a word for a role.
|
||||
|
||||
**Leaving the outcome as a reply to the asker's inbox.** Rejected on authority: a holder able to answer
|
||||
any asker needs a grant across the whole inbox space, which is the one grant design 25 §4 refuses by
|
||||
name. The seat's event costs the asker a filter and costs the mesh nothing.
|
||||
|
||||
## Reconciled with 0122, which landed in parallel
|
||||
|
||||
*Added 2026-09-27, on merging.* [ADR 0122](0122-a-seat-is-data-a-rename-is-a-database-update.md) moved
|
||||
the seat set out of compiled code and into a table the controller owns. This record was written against
|
||||
the slice, and says the `mesh-*` set "gains the same three fields a declared seat has".
|
||||
|
||||
**The decision is unaffected and the mechanism is better for it.** What a seat accepts, emits and serves
|
||||
becomes three columns beside its name and scope, so giving a role a protocol is a write rather than a
|
||||
rebuild — which is the whole argument of 0122 applied to the thing this record adds. Where this text
|
||||
says the set gains fields, read: the table gains columns.
|
||||
|
||||
## Consequences
|
||||
|
||||
**A seat is now the mesh's unit of "a role that talks".** A role that accepts work, announces outcomes
|
||||
or answers questions says so where it is defined, and everything about permissions, queues and
|
||||
consumers follows. Nothing hand-writes a grant for a role again.
|
||||
|
||||
**The shared library cannot yet publish on a seat, and that is now the blocking gap rather than a
|
||||
curiosity.** A module holding a seat has the authority and no way to use it; the build machine is
|
||||
written in Go and reaches the bus directly, so it is unaffected, but the artifact-store event stays
|
||||
under its module's own name until the library has a surface for this. That is a task, and this record
|
||||
is what makes it one.
|
||||
|
||||
**A second mechanism disappears.** `mesh.build.*`, the BUILDS stream and the builder's hand-written
|
||||
account all retire. Fewer things, and the ones left are derived.
|
||||
|
||||
**The catch-up question is not settled by this**, only made answerable: a seat that serves something
|
||||
gives the controller a way to be asked, which the mesh did not have. Whether catch-up should be a
|
||||
question at all remains open.
|
||||
@@ -0,0 +1,71 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
---
|
||||
|
||||
# 130. The predecessor is ending, and its broker goes with it
|
||||
|
||||
|
||||
> **Pointer repointed, 2026-09-27.** This record was written extending
|
||||
> [ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)), which
|
||||
> [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md) has since superseded — AMQP is
|
||||
> not a provision at all. Nothing decided here changes; the frontmatter now rests on the live record,
|
||||
> and the citations below are read with that in mind.
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)) settled that the old broker is an ordinary
|
||||
provider of the `amqp` provision rather than a compatibility module with an end date. It rejected
|
||||
giving it a retirement condition, and said why: *"its clients are not only the predecessor's, so the
|
||||
retirement condition describes a day that will not come."*
|
||||
|
||||
**The operator has said that day is coming.** The predecessor is deprecated. Some of it is still
|
||||
running, and it is not being migrated — it is being left to stop. Its broker may be shut down.
|
||||
|
||||
That is a fact about this installation, not a change of mind about what a broker is. It is recorded
|
||||
because three documents reason from the premise it overturns:
|
||||
[design 25](../03-DESIGN/01-to-be/25-the-bus-on-nats.md) §5 and §9, and
|
||||
[design 28](../03-DESIGN/01-to-be/28-building-the-bus.md)'s closing note that the predecessor's world
|
||||
"does not need to move: its broker is the compatibility module until its last client is gone."
|
||||
|
||||
## Decision
|
||||
|
||||
**The predecessor's broker retires when nothing requires `amqp`, by being unassigned like any other
|
||||
provider.** No retirement condition, no end-date machinery, no special case — which is ADR 0127 being
|
||||
paid off rather than revised. Because that record made the broker an ordinary provider, ending it
|
||||
needs nothing that does not already exist: a provision with no consumers has its provider unassigned,
|
||||
and the module system has done that since it existed.
|
||||
|
||||
**So step 5.3 has an ending.** "The mesh's own accounts removed from the deprecated broker" was
|
||||
written as the last thing that could be said, because the broker itself was going to outlive the
|
||||
question. It now finishes: once the mesh's own traffic has moved and the predecessor's remnants have
|
||||
stopped, the module is unassigned and the port is free.
|
||||
|
||||
**And the transitional doubling has a date.** The build outcome is announced under both the module's
|
||||
name and the role's on the old bus, so that a catalogue deployed before the rename and one deployed
|
||||
after both hear it. That exists only while the old bus does, and goes with it.
|
||||
|
||||
## Consequences
|
||||
|
||||
**The remote access path goes with it, and that is the one practical consequence worth planning
|
||||
around.** The predecessor's own mesh communicates over that broker — so shutting it down ends the
|
||||
tooling that reaches this installation's machines remotely. Work on the node after that point is done
|
||||
from the node. **This matters most for the rollout**, which is the step that would otherwise be driven
|
||||
from a workstation: it has to be driven locally, or driven before the broker stops.
|
||||
|
||||
**What is still running on it stops when it stops.** Some of the predecessor's services are live and
|
||||
are not being moved. That is the operator's decision and it is recorded here so that nobody later reads
|
||||
a broker with clients as an accident.
|
||||
|
||||
**Nothing in a served request's path is affected.** Modules serve from their own containers; the mesh's
|
||||
bus carries the mesh's own traffic — declarations, reports, events, tool calls. This was checked rather
|
||||
than assumed when the question came up, and it is why the operator's position (*"as long as my services
|
||||
keep running"*) is a bounded risk rather than a gamble.
|
||||
|
||||
**One reason to keep the broker survives**: `amqp` remains a provision a module may require, and a
|
||||
module that genuinely needs an AMQP broker can be given one. What retires is *this* broker's role as
|
||||
the predecessor's, not the mesh's ability to provide the thing.
|
||||
@@ -0,0 +1,94 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-27
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
supersedes: 0127-amqp-is-a-provision-not-the-bus.md
|
||||
---
|
||||
|
||||
# 131. Everything on the mesh speaks to the broker seat, and AMQP is not a provision
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0127](0127-amqp-is-a-provision-not-the-bus.md) settled the old broker as an ordinary provider
|
||||
of an ordinary provision, `amqp`, kept for whatever wanted a message broker of its own. The day the
|
||||
bus moved was the day that framing was tested, and it failed in a way that took the control plane
|
||||
down for an evening.
|
||||
|
||||
Three things came out of the wreckage. **The protocol had leaked into the seat's contract**: for a
|
||||
module to hold `mesh-broker`, it had to provide what the seat delivers, and what it delivered was
|
||||
`amqp` — so the module that will carry the bus on NATS could not hold the seat that names the bus,
|
||||
while the module the mesh was leaving could. **A consumer of `amqp` is not asking for AMQP.** The two
|
||||
modules requiring it wanted the mesh's messaging — to emit an event, to hear a topic — and named the
|
||||
wire protocol only because that was the word available. **And AMQP and NATS are not interchangeable
|
||||
at the wire.** A provision named after a protocol can only ever be answered by that protocol, so once
|
||||
the bus is NATS an `amqp` provision has one possible provider, and it is the thing being retired.
|
||||
|
||||
The operator's position, stated during the outage: modules depend on the broker *seat*, not on a
|
||||
protocol; AMQP is obsolete as anything the mesh's core knows about; a module that depends on `amqp`
|
||||
is wrong; and everything should reach the mesh's bus and be able to emit events and consume topics
|
||||
through it.
|
||||
|
||||
## Decision
|
||||
|
||||
**A module that needs messaging uses the mesh's bus, and the mesh's bus is whatever holds
|
||||
`mesh-broker`.** Emitting an event and consuming a topic go through the sdk, which is handed the
|
||||
bus by the mesh with the module's own credential. No manifest names a wire protocol to get it.
|
||||
|
||||
**`amqp` is neither a provision nor a requirement.** Registration refuses a manifest that provides
|
||||
it or requires it. The `mesh-broker` seat delivers `mesh-bus`, and its holder is the module that
|
||||
provides `mesh-bus` — today the nats module, and only it.
|
||||
|
||||
**The old broker's module and the two modules that required it leave the catalogue.** They are
|
||||
removed, not converted: one was a proof that a grant worked end to end, the other forwards mail off a
|
||||
queue, and both are re-done against the bus if wanted, as new modules under this record.
|
||||
|
||||
**The controller's AMQP transport is deleted once every node reports on the new bus**, and the
|
||||
switch that selects a transport goes with it — one bus, so nothing to select.
|
||||
|
||||
The predecessor's own broker is outside the mesh and not this record's concern
|
||||
([ADR 0130](0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md)): what the predecessor's
|
||||
tooling loses when it stops is accepted there.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **Keep 0127: AMQP stays an ordinary provision with the old broker as its provider.** Rejected. It
|
||||
is what put the protocol into the seat's contract, it is why the seat could be left with no valid
|
||||
holder mid-change, and it keeps two transports in the control plane indefinitely for the benefit of
|
||||
two modules that did not want AMQP in the first place.
|
||||
2. **Bridge it: the old broker's module also provides `mesh-bus`, so both can hold the seat during the
|
||||
change.** Rejected. It makes the retiring broker a legitimate mesh bus for exactly as long as
|
||||
nobody removes the line, which in practice is forever, and it leaves `amqp` as a thing the core
|
||||
still knows the name of.
|
||||
3. **The seat is the dependency; the protocol is nobody's business but the holder's.** Adopted.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **The change of holder is a handover, and it needs a command.** Nothing today moves a seat from
|
||||
one assignment to another as one act, and a seat the control plane dereferences cannot be empty
|
||||
in between — that emptiness is the outage this record comes from. The command takes a seat and the
|
||||
assignment taking it over. Designed and built before the cutover, under
|
||||
[28 — Building the bus](../03-DESIGN/01-to-be/28-building-the-bus.md).
|
||||
- **The seat's row moves to `mesh-bus` before the new holder registers, and that is safe.** The
|
||||
control plane composes its own bus address through the seat *by name*
|
||||
(`${seat:mesh-broker:…}`), and the overview derives holders by name; only registration and the
|
||||
provision-to-seat resolution read what a seat delivers. So the row can change under the current
|
||||
holder without unseating it, the new holder can then register its claim, and the handover happens
|
||||
when both are running. Verified in the code during the outage, not assumed.
|
||||
- **Registration gains two refusals**: a manifest providing `amqp`, and one requiring it.
|
||||
- **The `rollout check` stops saying the old broker stays.** It said so under 0127; it now lists
|
||||
unassigning it as the last step of the move.
|
||||
- **What got harder**: a third party that genuinely wants an AMQP broker on a mesh node runs one as
|
||||
any application module, with no provision and no seat, and nothing on the mesh routes to it. That
|
||||
is the cost of the mesh not knowing the word.
|
||||
|
||||
## How this is checked
|
||||
|
||||
| Rule | Checked by |
|
||||
|---|---|
|
||||
| No manifest provides or requires `amqp` | a registration test refusing each, naming this record; and a whole-catalogue test asserting no registered manifest names it |
|
||||
| `mesh-broker` delivers `mesh-bus`, and only a `mesh-bus` provider may hold it | the existing registration test for a delivering seat, with the row's value read from the store (mesh-controller#89) |
|
||||
| The seat's row can change without unseating the holder | a test composing the control plane's own address and the overview under a row that the current holder does not satisfy |
|
||||
| The rollout does not leave the old broker running | `rollout check` output, asserted in its test |
|
||||
| The AMQP transport is gone | the package does not compile with it referenced; the switch variable is refused as unknown at start |
|
||||
@@ -0,0 +1,150 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md
|
||||
---
|
||||
|
||||
# 132. A seat carries the tools its holder must serve
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0129](0129-a-seat-carries-the-protocol-of-its-role.md) gave a seat the protocol of its role in
|
||||
three parts: the work it accepts, the events it emits, and the verbs it **serves** — request and
|
||||
reply, awaited. The bus already derives authority from all three: a holder subscribes
|
||||
`mesh.seat.<seat>.tool.<verb>`, and a module that uses the seat may publish it and nothing else.
|
||||
|
||||
**The serving third has never been used.** The mesh defines 14 seats, 8 mesh-scoped and 6
|
||||
node-scoped. Exactly one carries a protocol at all — the build machine, which accepts `build` and
|
||||
emits `built`. Not one seat declares a single verb it serves. The mechanism is built, enforced, and
|
||||
empty.
|
||||
|
||||
Meanwhile every tool on the mesh is addressed to a module. Of 72 modules in the catalogue, 45 serve
|
||||
tools, about 203 of them, each on `mesh.mod.<module>.tool.<name>`. So a caller binds to the module
|
||||
that happens to hold a role rather than to the role, and replacing that module breaks every caller —
|
||||
which is the thing seats exist to prevent everywhere else.
|
||||
|
||||
**Nothing can say what tools exist.** Measured on 2026-09-28, with the bus carrying the whole mesh: a
|
||||
workstation client holding an operator credential connected, the bus accepted the account, and
|
||||
`mesh call gitea.gitea_list_repos` answered with real repositories. The same client's `mesh tools`
|
||||
found nothing, because it asks `mesh-catalog.catalog_tools` and no module serves that: the catalogue
|
||||
serves `catalog_modules`, `catalog_module`, `catalog_provides`, `catalog_dependents` and
|
||||
`catalog_stale`. An agent can therefore call any tool it already knows the name of and discover none.
|
||||
MCP's `tools/list` is that same question, so the MCP surface is a working transport over an empty
|
||||
catalogue.
|
||||
|
||||
**And there is nowhere for a tool's definition to live.** A manifest has a `tools` field: 0 of the 45
|
||||
modules that serve tools fill it. That is not neglect, it is the arrangement failing — the field was
|
||||
the bus grant's source for what a module may subscribe, and because nothing filled it every module
|
||||
that served a tool was refused its own subscription on the new bus, live, until the grant was changed
|
||||
to the module's own namespace. Today a tool's name, description and argument schema exist only in the
|
||||
module's code.
|
||||
|
||||
Two facts about the machinery matter for what follows. A seat's protocol is not in the store: the seat
|
||||
rows lack the ADR 0129 columns, so the protocol comes from compiled defaults and is merged in when a
|
||||
row is read. And `seatSubject` is flat — `mesh.seat.<seat>.<kind>.<verb>` with no node in it — so a
|
||||
node-scoped seat's tool call would reach every node's holder at once, and the holders' queue group
|
||||
would hand it to whichever answered first.
|
||||
|
||||
## Decision
|
||||
|
||||
**A seat's protocol carries its tools in full**: the verb, what it does, and the schema of its
|
||||
arguments and of its answer. The seat is the definition of the role's interface; the holder is an
|
||||
implementation of it.
|
||||
|
||||
**Serving the seat's tools is a condition of holding the seat.** A module that does not serve every
|
||||
verb the seat declares may not occupy it. This is checked where the other conditions of holding are
|
||||
checked — registration and handover — and refused by naming the verbs that are missing.
|
||||
|
||||
**A role's tools are addressed to the role.** `mesh.seat.<seat>.tool.<verb>` mesh-wide. A node-scoped
|
||||
seat carries the node in the address, because one subject reaching six machines' holders is not an
|
||||
address, and the queue group that made it look like one would silently pick a winner.
|
||||
|
||||
**A module keeps its own tools, and both exist.** `gitea_list_repos` stays, because gitea can run
|
||||
without holding the `git` seat — a second forge, an instance kept for one purpose. The module's name
|
||||
answers *this gitea*; the seat's verb answers *whoever is the forge*. Which of the two a caller wants
|
||||
is a decision in the running session, not one the mesh makes for it.
|
||||
|
||||
**What answers "what tools exist" follows where the definition lives.** A seat's tools are read from
|
||||
the mesh's own records. A module's own tools are answered by the module, from the code that defines
|
||||
them. Discovery is therefore a read for the durable half and a question to the running mesh for the
|
||||
free half.
|
||||
|
||||
**A seat's tools are an interface, and change like one.** Additive within a version; a change that
|
||||
would break a caller takes the version token the subject already has room for (design 29 §8), and the
|
||||
two run side by side until nothing is bound to the old one.
|
||||
|
||||
**The mesh's own verbs are the `mesh-controller` seat's tools.** `status`, `push`, `build`, `assign`
|
||||
and the rest are a role's interface, not a container's, and the audit point [ADR 0095](0095-the-control-plane-is-the-way-to-ask-a-module.md)
|
||||
asks for is the seat's holder.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **The manifest declares each module's tools.** Rejected. The list is then written twice — in the
|
||||
manifest and in the code — and a schema in a manifest goes stale silently, which is the worst kind
|
||||
of wrong for something an agent reads to decide what to call. It is also the arrangement that has
|
||||
already failed once: the field exists, 0 of 45 modules fill it, and the grant that depended on it
|
||||
refused every tool subscription on the mesh.
|
||||
2. **Every runtime answers an introspection call, and something aggregates them.** Rejected as the
|
||||
shape for a role's tools, kept for a module's own. An aggregator needs permission to publish into
|
||||
every module's namespace, which is a widening the mesh otherwise gives only to the control plane;
|
||||
and the answer is only as available as the modules are, so a mesh whose catalogue cannot say what a
|
||||
role answers while its holder is down cannot plan against it.
|
||||
3. **The control plane answers everything.** Rejected. It puts a tool surface on the control plane for
|
||||
tools it does not implement, and makes discovery depend on the one component that must stay
|
||||
answerable while it is itself being replaced. The mesh's own verbs are its to answer, and it answers
|
||||
them as the holder of a seat.
|
||||
4. **Seats only; no module tools.** Rejected. Most modules hold no seat, and inventing a seat per
|
||||
module to give its tools a home would dilute what a seat is: one holder of a role the mesh needs
|
||||
exactly one of.
|
||||
|
||||
## Consequences
|
||||
|
||||
**One capability can have two names, deliberately.** A forge that holds the `git` seat answers both
|
||||
`mesh.seat.git.tool.list_repos` and `mesh.mod.gitea.tool.gitea_list_repos`. This is the one place the
|
||||
mesh accepts two names for one thing, because they are answers to different questions and the second
|
||||
one survives the module not holding the seat. The glossary rule stands everywhere else.
|
||||
|
||||
**A seat becomes a contract to implement.** Adding a verb to a seat is a change every holder must
|
||||
make, and a claim that was valid becomes invalid until it does. That is the point, and it is also the
|
||||
reason a seat's tools should be few and durable while a module's own stay free.
|
||||
|
||||
**Three prerequisites, none of them in place.** The seat's protocol must be in the store rather than in
|
||||
compiled defaults, or discovery reads a binary rather than the mesh. The protocol must become richer
|
||||
than a list of verbs, because a verb without a schema is not something an agent can call. And a
|
||||
node-scoped seat needs the node in its subject before any of its tools can exist.
|
||||
|
||||
**Discovery becomes cheap for the half that matters.** What roles the mesh has and what each answers is
|
||||
a query, with no fan-out and nothing to be up. An agent's authority can then be role-shaped — *the
|
||||
forge's tools* — rather than a list of module-specific names that changes when a module is replaced.
|
||||
|
||||
**The MCP surface belongs inside the mesh.** Once the tools are the mesh's own records, the thing that
|
||||
serves them to an agent is a module the mesh assigns to the machine where the agent sits, with a
|
||||
credential the mesh minted and authority derived from what it may call — not a program started by hand
|
||||
with a credential printed to a terminal.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **Holding is refused without the verbs.** The condition sits with the other conditions of holding a
|
||||
seat, so registration and a handover both refuse a module that does not serve what the seat declares,
|
||||
and the refusal names the missing verbs. A test per condition, as the other seat conditions have.
|
||||
- **The grant is derived from the seat, and already is.** A holder's subscription and a user's publish
|
||||
come from the seat's protocol, so a verb nobody declared is a subject nobody may use, and a verb the
|
||||
seat declares reaches exactly its holder. The golden composition of the bus's user list is the test
|
||||
that keeps it honest.
|
||||
- **Discovery is a read, and is tested as one.** What the mesh answers for a seat's tools equals what
|
||||
the seat's records declare — no call to a module in the path, so the test needs no running module.
|
||||
- **A node-scoped seat's subject carries its node**, checked by the same test that checks the subject
|
||||
table: two nodes holding one node-scoped seat derive two addresses.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0129](0129-a-seat-carries-the-protocol-of-its-role.md) — the protocol this widens
|
||||
- [ADR 0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md) — the role, its work and its events
|
||||
- [ADR 0095](0095-the-control-plane-is-the-way-to-ask-a-module.md) — a tool call passes one process where an audit belongs
|
||||
- [ADR 0126](0126-a-module-declares-its-own-seats.md) — an event is addressed to its emitter, for the same reason a role's verb is addressed to its role
|
||||
- [`03-DESIGN/01-to-be/26-the-seats.md`](../03-DESIGN/01-to-be/26-the-seats.md) — how a seat is held and handed over
|
||||
- mesh-controller #116, #117, #118 — the grants as they now stand: a module serves its own namespace, the control plane may ask any tool
|
||||
- Measured 2026-09-28 on the live mesh: an operator credential calling a module's tool over the bus answers; `tools/list` finds nothing
|
||||
@@ -0,0 +1,159 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: superseded
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0052-a-step-that-runs-once-before-a-container.md
|
||||
superseded-by: 02-DECISIONS/0135-a-module-version-prepares-its-state-before-it-runs.md
|
||||
---
|
||||
|
||||
# 133. A module owns its migrations, and the mesh owns when they run
|
||||
|
||||
## Context
|
||||
|
||||
On 2026-09-28 the mesh replaced its own control plane, through its own upgrade path, with a build
|
||||
carrying a migration. Nothing applied it. For the next three quarters of an hour every build the mesh
|
||||
made was refused by the store with one line — *column "built_contexts" does not exist* — which reached
|
||||
only whoever happened to be waiting on that build's reply. The images were built and published, so the
|
||||
registry filled with artifacts the mesh has no record of, and the overview went on reporting that every
|
||||
module was current ([issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md)).
|
||||
|
||||
The schema had been created once, at genesis, by an action in the foundation bundle. Nothing ran it
|
||||
again, through many updates of the control plane since.
|
||||
|
||||
**The mechanism to do this right already existed and one module used it wrong.**
|
||||
[ADR 0052](0052-a-step-that-runs-once-before-a-container.md) makes a run-once container a step the host
|
||||
runs to completion before whatever the declaration places after it, and names migrating a schema as the
|
||||
case it exists for. Three facts about how it is used today:
|
||||
|
||||
- The control plane's manifest had no step at all. The immediate fix was to write one by hand, and that
|
||||
hand-written step repeats three environment variables and three volume mounts from the server
|
||||
resource it precedes — six chances to drift from the thing it prepares.
|
||||
- Two other modules hand-write the same shape for the same reason: gitea's admin bootstrap and
|
||||
mosquitto's dynsec seed, each repeating its sibling's image, environment and mounts. One of them
|
||||
ends in `|| true`, which is a lock implemented as a shrug.
|
||||
- The catalogue module takes the other road: it migrates its own schema in its own code when it starts.
|
||||
That failure mode is a crash loop rather than a stop — the catalogue restarted 338 times this
|
||||
morning on an unrelated start-time failure, and nothing anywhere said the mesh's graph had a gap.
|
||||
|
||||
**What the mesh already has, and what HAL needed stages for.** Ordering a provider before its consumer
|
||||
is `providersFirst`, which topologically orders a node's modules. Ordering within a module is
|
||||
declaration order, and a run-once container gates everything after it. Remembering that a step has
|
||||
already run is the digest of its declaration, recorded only after it exits 0
|
||||
([ADR 0018](0018-a-picture-is-read-from-what-runs.md)) — and the image is part of that digest, so a new
|
||||
build re-runs it. Three of the four things a stage system provides are therefore already here. The
|
||||
fourth — that a module has a schema at all — is the only thing missing.
|
||||
|
||||
**Nothing in the catalogue ships a migrations directory.** Of 72 modules, none has one; the modules that
|
||||
migrate do it in their own code. So this is not a decision about where SQL files live. It is a decision
|
||||
about who runs them and when.
|
||||
|
||||
**Two facts bound what is safely expressible.** A node converges toward its own declaration without
|
||||
waiting on any other node. And of the five modules that run on more than one machine today — dnsmasq,
|
||||
fail2ban, networking, networkmanager, sshd — not one wants a store; every module with a database is on
|
||||
exactly one machine.
|
||||
|
||||
## Decision
|
||||
|
||||
**A container may declare steps to run before it.** The same container, run to completion, with
|
||||
different arguments, in order, before it starts. The mesh derives the run-once resources from that
|
||||
declaration, so the image, the environment, the volumes, the network and the credentials come from the
|
||||
one place they are already described and cannot drift from it.
|
||||
|
||||
**A module's migrations are the first user of this, and the module owns them entirely.** The SQL, the
|
||||
order, the idempotence, the lock, and which dialect it speaks. The mesh never learns that postgres and
|
||||
mssql differ, because it runs the module's own image with the module's own arguments against the
|
||||
module's own binding and requires exit 0. A module needing both stores runs one step that does both.
|
||||
|
||||
**The mesh owns the moment, and the gate is the guarantee.** Whether a version may serve when its
|
||||
schema is not there yet is a deployment question, and the mesh is the only thing that can answer it,
|
||||
because the mesh is what starts the container. A step that fails stops the container it precedes, so
|
||||
a failed migration is a version that does not serve rather than a version serving against a store it
|
||||
does not match.
|
||||
|
||||
**Per node, and there is no level.** The step runs wherever the module runs. A step that ran "once,
|
||||
somewhere" would leave every other machine with no gate at all, and additive migrations protect old
|
||||
code against a new schema, never new code against an old one. The cost is an obligation a migration
|
||||
runner already carries: a version table and a lock.
|
||||
|
||||
**"Once, mesh-wide" is what holding a seat means.** A step that is not idempotent — seeding an
|
||||
account, sending a notice, taking a backup — belongs to a module that holds a seat, where the mesh
|
||||
already guarantees one holder, on record, handed over deliberately. That is the answer to the level
|
||||
question rather than a field that has to invent an election and keep it somewhere.
|
||||
|
||||
**Migrations are forward-only and additive.** The step runs before the *new* container starts, so the
|
||||
old one is still running against the new schema for the length of the apply.
|
||||
|
||||
**Declared, never inferred.** The control plane cannot see inside an image, so a module that ships
|
||||
migrations and declares no step is not refusable at registration; it breaks on its first upgrade. This
|
||||
record says so rather than implying a check that cannot exist.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **Each module migrates itself when it starts** — what the catalogue does today. Rejected: it turns a
|
||||
schema failure into a crash loop instead of a stop, it is invisible in the declaration so nothing can
|
||||
say the module even has a schema, and two machines running the module both migrate at start with
|
||||
nothing sequencing them.
|
||||
2. **The mesh applies migrations itself**, with a driver and a version table per store — HAL's shape.
|
||||
Rejected: the mesh would have to know one store type from another, hold another module's store
|
||||
credentials, and reach a machine to use them, which [ADR 0005](0005-the-node-host.md) forbids. It is
|
||||
also the reason that shape needs levels: something central has to decide where the once happens.
|
||||
3. **A hook lifecycle** — pre-build, post-build, pre-deploy, post-deploy. Rejected: there is no deploy
|
||||
event here to hook. A declaration is a desired state applied in order and reconciled forever, so
|
||||
"pre-deploy" is exactly "a step before this container", pre- and post-build are what a Dockerfile and
|
||||
the artifact list already are, and "post-deploy" has no moment to name.
|
||||
4. **A hook level** — once per module, or once per module-node assignment. Rejected as a field, kept as
|
||||
a property: see the decision. A once-per-module step needs cross-node ordering underneath it to be
|
||||
safe, and a node converging without waiting on its neighbours is worth losing on purpose rather than
|
||||
by accident.
|
||||
5. **Every module hand-writes its own run-once step** — the immediate fix for the control plane.
|
||||
Rejected as the general answer: it duplicates the resource it precedes, in three places already, and
|
||||
a hand-written step is one the next module forgets. Forgetting it is the fault this record exists
|
||||
for.
|
||||
6. **Record a schema level per module in the store.** Rejected: gating makes the invariant true by
|
||||
construction, so a level is a second account of the same fact and the first one to go stale.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Three hand-written steps collapse into one line each**, and the control plane's own migrate step stops
|
||||
repeating its server's environment and mounts.
|
||||
|
||||
**The catalogue's self-migration becomes the exception to remove.** One shape, and the mesh's own
|
||||
control plane is not an exception to it either.
|
||||
|
||||
**A module on two machines with one shared store must lock.** Today none is, so this is an obligation
|
||||
stated before it is needed rather than discovered by two concurrent migrations.
|
||||
|
||||
**There is still no readiness-gated step.** Only an action carries `verify`; a container has no health
|
||||
notion, so "run this once the service answers" remains unexpressible and seeding through a running
|
||||
service's API has no home. That is its own decision about a container's readiness, and this record does
|
||||
not make it.
|
||||
|
||||
**Genesis keeps its own action.** At birth there is no control plane to derive anything from, which is
|
||||
what [ADR 0067](0067-genesis-is-a-pivot.md) already says about that moment.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **The composition carries the step.** A test on a node's composed declaration: every container that
|
||||
declares steps before it is preceded by them, and the derived step's image, environment, volumes and
|
||||
network equal the container's — so the two cannot drift, which is the failure the hand-written kind
|
||||
has.
|
||||
- **A failed step stops what follows.** The host already refuses to go on past a run-once step that did
|
||||
not exit 0; the test for that is extended to a derived one, so the gate is checked rather than
|
||||
assumed.
|
||||
- **The mesh's own schema is covered by the same mechanism as everything else.** The control plane
|
||||
declares its step in its own manifest, so the case that failed on 2026-09-28 is the case the test
|
||||
covers.
|
||||
- **A module claiming a seat for a once-only step is checked where seats are checked** — the conditions
|
||||
of holding, not a new mechanism.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0052](0052-a-step-that-runs-once-before-a-container.md) — the step this extends
|
||||
- [ADR 0018](0018-a-picture-is-read-from-what-runs.md) — a digest is the record that something happened
|
||||
- [ADR 0005](0005-the-node-host.md) — the control plane decides and never touches a machine
|
||||
- [ADR 0067](0067-genesis-is-a-pivot.md) — why genesis does it differently, once
|
||||
- [issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md) — the failure that produced this record
|
||||
- [`03-DESIGN/01-to-be/32-what-a-module-declares.md`](../03-DESIGN/01-to-be/32-what-a-module-declares.md) §6 — the lifecycle this sits in
|
||||
- Measured 2026-09-28: three hand-written run-once steps repeating their sibling's resource; 0 of 72 modules with a migrations directory; 5 modules on more than one machine, none of them wanting a store
|
||||
@@ -0,0 +1,126 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
---
|
||||
|
||||
# 134. The mesh says what it applied
|
||||
|
||||
## Context
|
||||
|
||||
The pipeline is observable on the bus from a merge to an artifact, and modules already plug into it:
|
||||
the forge emits `pull.merged`, the build machine's seat emits `built`, the catalogue emits `registered`,
|
||||
`upgraded` and `rebuild-needed`, providers emit `postgres.database.provisioned` and its siblings. Things
|
||||
consume them today — the catalogue consumes `built`, each provider consumes its own provisioning events,
|
||||
model-usage consumes `*.usage.*`, the audit logger consumes `**`. Nothing had to be invented for any of
|
||||
that; subscribing *is* plugging in.
|
||||
|
||||
**It goes dark at the moment it touches a machine.** A host applies a declaration and reports to the
|
||||
control plane on the control branch, which only the control plane may read — correctly, because a report
|
||||
carries what a machine is and enrolment travels the same way. So nothing on the mesh says *this machine
|
||||
now runs version Y of module Z*, or that it refused to, or why.
|
||||
|
||||
What that cost on 2026-09-28, in one morning:
|
||||
|
||||
- A build result the store refused was visible only to whoever was waiting on that build's reply. For
|
||||
three quarters of an hour the mesh built things and recorded none of them, while the overview said
|
||||
every module was current ([issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md)).
|
||||
- A module crash-looping at start — 338 restarts — was found by reading a container's logs by hand.
|
||||
Nothing on the bus said the mesh's graph had stopped learning.
|
||||
- A run-once step that fails now stops an upgrade by design ([ADR 0133](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md)),
|
||||
and the same silence would cover it: the version simply would not appear.
|
||||
|
||||
**And the one thing the control plane does emit is refused by its own account.** Answering a catalogue
|
||||
that asks to catch up, it publishes each recorded build under `mesh.mod.control-plane.event.…` — a
|
||||
module namespace for a module that does not exist. Its own permissions refuse it, so a catalogue that
|
||||
restarts gets nothing and keeps its gap. The control plane has facts to state and nowhere to state
|
||||
them.
|
||||
|
||||
## Decision
|
||||
|
||||
**The mesh emits the deploy half of the pipeline as facts on the bus.** What a machine now runs, and
|
||||
what it refused to run and why. Both are facts about the mesh doing its work, in the same form as every
|
||||
other fact on the bus, so anything that wants them subscribes the way the catalogue subscribes to
|
||||
`built`.
|
||||
|
||||
**The control plane states them, as the holder of the `mesh-controller` seat.** Its facts live under the
|
||||
seat's own namespace, which is where a role's events belong
|
||||
([ADR 0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md),
|
||||
[ADR 0129](0129-a-seat-carries-the-protocol-of-its-role.md)) and which survives the control plane being
|
||||
replaced. That is also what gives the catch-up replay a subject it may publish instead of an invented
|
||||
module namespace.
|
||||
|
||||
**Emitted when what a machine runs changes, not on every convergence pass.** A host reconciles
|
||||
continuously and reports each time; a fact per pass would be a fact per minute per machine that says
|
||||
nothing. The report carries the declaration it applied and what changed, so the control plane has what
|
||||
it needs to speak only when there is something to say.
|
||||
|
||||
**A refusal is a fact with a subject in it** — which machine, which resource, and the reason as the host
|
||||
gave it. A refusal that names only the machine is the silence this record is about, one level up.
|
||||
|
||||
**Reports stay where they are.** A node's report remains control traffic that only the control plane
|
||||
reads. The deploy facts are derived from it, which makes them second-hand on purpose: one emitter, one
|
||||
ordering, and no widening of the narrowest account in the mesh.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **Leave it as it is, and let whatever cares ask the control plane.** Rejected: asking for a fact that
|
||||
already arrives is the shape the mesh removed everywhere else, and nothing can react at the moment a
|
||||
machine changes — which is exactly when a graph, an audit or an operator wants to know.
|
||||
2. **Each node emits its own facts.** Rejected: it widens every host's account to an event namespace, and
|
||||
a host's authority is deliberately the narrowest in the mesh. Its report already reaches the one thing
|
||||
that can speak for it.
|
||||
3. **Widen who may read the control branch.** Rejected: that branch carries what machines say *to* the
|
||||
control plane, enrolment included. Widening its readers widens that too, for an unrelated reason.
|
||||
4. **A registry of deploy hooks** — something registers interest and is called. Rejected: an event is
|
||||
already the mechanism; there is nothing to register, and a callback is an address the mesh spent
|
||||
[issue 102](../04-ISSUES/102-an-address-recorded-at-genesis-or-build-does-not-follow-the-nodes-ports/00-report.md)
|
||||
learning not to keep.
|
||||
5. **Put the facts on the node's own declaration stream.** Rejected: that stream is last-per-subject by
|
||||
design, so a machine away for an hour gets exactly the current declaration and nothing older. A
|
||||
history of what happened cannot live in a stream built to forget.
|
||||
|
||||
## Consequences
|
||||
|
||||
**The audit logger gets the deploy half for nothing**, because it consumes everything.
|
||||
|
||||
**A failure becomes visible where the mesh is watched** rather than where someone happened to be
|
||||
looking. That answers the open question [issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md)
|
||||
left about a record the store refused.
|
||||
|
||||
**The catch-up replay stops being a burst of events.** With the control plane able to state its own
|
||||
facts, replaying history as if it were happening now is a choice rather than the only option — and the
|
||||
better shape is the question the catalogue is actually asking, answered once
|
||||
([design 33](../03-DESIGN/01-to-be/33-the-tools-the-mesh-answers.md)).
|
||||
|
||||
**The facts are second-hand.** The control plane says what a machine reported, so a machine that cannot
|
||||
reach the bus produces no fact at all. Absence is not health, and what a machine was last heard from
|
||||
stays the place that says so.
|
||||
|
||||
**The events stream carries more.** Bounded by emitting on change rather than on every pass, and each
|
||||
fact is small; the stream's own limits remain what keeps it finite.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **The control plane's grant names exactly the subjects it emits**, derived from its seat like every
|
||||
other principal's, and the composed user list is compared against a golden file — so a fact it cannot
|
||||
publish fails a test rather than a catalogue's replay.
|
||||
- **A convergence that changed nothing emits nothing.** A test with two identical reports and one
|
||||
expected fact, because the failure this guards against is a fact per minute per machine.
|
||||
- **A refusal names its resource.** A test where a host reports a failed resource and the emitted fact
|
||||
carries which one and why, not merely that something went wrong.
|
||||
- **What the mesh emits is what something consumes.** The subject a module declares it consumes derives
|
||||
to the subject the control plane publishes — the same agreement test that already keeps the
|
||||
controller's own subscriptions honest.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0041](0041-events-are-a-relationship.md) — an event is a relationship, not a call
|
||||
- [ADR 0126](0126-a-module-declares-its-own-seats.md) — an event is addressed to its emitter, because the emitter's identity is the meaning
|
||||
- [ADR 0121](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md), [ADR 0129](0129-a-seat-carries-the-protocol-of-its-role.md) — a role's events belong to the role
|
||||
- [ADR 0083](0083-one-push-leaves-the-mesh-consistent.md) — a report is held for the store rather than lost
|
||||
- [ADR 0133](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md) — the gate whose failure this makes visible
|
||||
- [`03-DESIGN/01-to-be/32-what-a-module-declares.md`](../03-DESIGN/01-to-be/32-what-a-module-declares.md) §6 — the lifecycle, which ends today at a report nobody else may read
|
||||
- Measured 2026-09-28: 45 minutes of builds recorded nowhere with the overview reporting health; a module at 338 restarts found by hand; the control plane's only emitted event refused by its own permissions
|
||||
@@ -0,0 +1,153 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
supersedes: 0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md
|
||||
---
|
||||
|
||||
# 135. A module version prepares its state before it runs
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0133](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md) settled who runs a
|
||||
module's migrations and when, and it said so in the wrong vocabulary. It put the declaration on a
|
||||
*container* — "a container may declare steps to run before it" — and derived the scope of the work from
|
||||
the *machine*. Both are wrong at the level a module author works at, and the second is wrong on the
|
||||
facts.
|
||||
|
||||
**A container is one resource kind the host applies.** A module has code, state and a version; whether
|
||||
its artifact is an image, a bundle or something later is the mesh's business. The module-facing
|
||||
vocabulary for a module's own code already exists and has nothing to do with a container runtime: a
|
||||
module declares **entrypoints** — this file is my tools, this file is my provisioner — and the mesh runs
|
||||
them. A manifest that says "run this container with these arguments, and here are the volumes and
|
||||
environment again" has an author writing down the machine's business twice.
|
||||
|
||||
**And the scope is not the machine's to decide, because the mesh already decided what a state is.** A
|
||||
consumer is a module *on a machine* (migration 0015, from
|
||||
[issue 022](../04-ISSUES/022-one-credential-per-node-per-provision-not-per-module/00-report.md)):
|
||||
the mesh derives a login per consumer and the provider creates a database owned by exactly that login
|
||||
([ADR 0048](0048-a-provider-creates-the-credential-the-mesh-minted.md)). So a module on three machines is three
|
||||
consumers, three credentials and three databases. There is no shared state for two machines to race over,
|
||||
and ADR 0133's central caveat — that a module's migrations must take a lock because two machines might
|
||||
migrate at once — describes a situation the mesh does not currently produce.
|
||||
|
||||
That correction makes the whole "level" question HAL answered with stages disappear: the scope of
|
||||
preparation is the scope of the state, and the mesh knows it.
|
||||
|
||||
What the earlier record got right and this one keeps: the module owns the work, the mesh owns the moment,
|
||||
the gate is the guarantee, migrations stay forward-only, and none of it can be inferred from inside an
|
||||
artifact. What produced it also stands — the control plane was replaced with a build carrying a migration,
|
||||
nothing applied it, and for three quarters of an hour every build was refused by the store with one line
|
||||
that reached only whoever was waiting on a reply
|
||||
([issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md)).
|
||||
|
||||
## Decision
|
||||
|
||||
**A module version declares an entrypoint that prepares its state.** One name in the manifest, in the
|
||||
same vocabulary as the entrypoints it already declares for its tools and its provisioner. No container,
|
||||
no command line, no environment, no mounts — those are how a machine runs the module's code, and the
|
||||
module already said that once.
|
||||
|
||||
**The mesh runs it as it runs that module's own code, to completion, in the module's own context.** Every
|
||||
binding, credential and setting the module's code would receive, because it *is* the module's code. How a
|
||||
machine does that is the host's business and stays there: for an image artifact it is the step
|
||||
[ADR 0052](0052-a-step-that-runs-once-before-a-container.md) already defines, and a later kind of artifact
|
||||
changes the host, not the manifest.
|
||||
|
||||
**Preparation gates the version.** A version whose preparation did not succeed does not run — anywhere.
|
||||
Since the rollout already sends machines one at a time and stops at the first that does not take a
|
||||
version, a preparation that fails stops the rollout there, leaving every other machine on the version
|
||||
that works.
|
||||
|
||||
**Preparation is scoped to the state, and the mesh derives that scope.** State the mesh provisions is per
|
||||
consumer — a module on a machine — so preparation happens once per consumer. State the module keeps on
|
||||
the machine is per machine, which is the same answer. A module that holds an exclusive seat has one of
|
||||
itself, so its preparation happens once by definition. No level, no election, no cross-node ordering, and
|
||||
no lock obligation invented for a race the mesh does not create.
|
||||
|
||||
**Once per version per state.** A version bump attempts preparation once against each state it has; the
|
||||
module's own runner decides there is nothing to do, which is what a runner with a version table does
|
||||
anyway. A retry after a partial failure runs it again, so the work is the module's to make safe against
|
||||
that — the one obligation no design can remove.
|
||||
|
||||
**Forward-only and additive.** Preparation runs while the previous version is still serving, so a
|
||||
migration that removes or renames what the old code reads breaks the mesh in the window between the two.
|
||||
|
||||
**Declared, never inferred.** The control plane cannot see inside an artifact, so a module that ships
|
||||
migrations and declares no entrypoint is not refusable at registration. It breaks on its first upgrade,
|
||||
and this record says so rather than implying a check that cannot exist.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **A container declares steps before it** — [ADR 0133](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md).
|
||||
Superseded, not because the mechanism is wrong but because the *declaration* is in the wrong place: it
|
||||
makes every module author restate the machine's arrangement, and it ties a module's own lifecycle to
|
||||
one resource kind. The host-side mechanism it named is retained and is now an implementation detail.
|
||||
2. **Each module prepares itself when it starts** — what the catalogue does today. Rejected: a schema
|
||||
failure becomes a crash loop rather than a stop, nothing in the declaration says the module has a
|
||||
state to prepare, and the version serves the moment it starts rather than after the state is right.
|
||||
3. **The mesh applies migrations itself**, with a driver and a version table per store type. Rejected:
|
||||
the mesh would have to know one store from another, hold another module's credentials and reach a
|
||||
machine with them, which [ADR 0005](0005-the-node-host.md) forbids. It is also what forces a stage
|
||||
system: something central has to decide where the work happens.
|
||||
4. **A hook lifecycle** — pre-build, post-build, pre-deploy, post-deploy. Rejected: a declaration is a
|
||||
desired state reconciled forever, so there is no deploy moment to hook. "Pre-deploy" is exactly this
|
||||
record; pre- and post-build are what a recipe and the artifact list already are; "post-deploy" names
|
||||
nothing that happens.
|
||||
5. **A declared level** — once per module, or once per assignment. Rejected: the mesh already knows what a
|
||||
state is, so asking an author to choose is asking them to restate a fact the mesh holds, with a chance
|
||||
of contradicting it.
|
||||
6. **Record a preparation level per module in the store.** Rejected for the reason ADR 0133 gave and this
|
||||
record keeps: gating makes the invariant true by construction, and a level is a second account of the
|
||||
same fact.
|
||||
|
||||
## Consequences
|
||||
|
||||
**An author's whole contract is one line, once.** Write the migration in the module's code, name the
|
||||
entrypoint that runs it, and every later version rolls out as: build, prepare, run — with nothing
|
||||
per-version to remember and nothing about the machine to restate. That is the property this exists for.
|
||||
|
||||
**Three hand-written steps in the catalogue collapse**, and the control plane's own migrate step stops
|
||||
repeating its server's environment and mounts.
|
||||
|
||||
**The catalogue's self-preparation becomes the exception to remove.** One shape, and the mesh's own
|
||||
control plane is not an exception either.
|
||||
|
||||
**A module scaled across machines with one shared state is not expressible**, and this record does not
|
||||
make it so. The mesh gives each consumer its own state; a deliberately shared one is a different
|
||||
provision model, and the place the "once, mesh-wide" question would genuinely return. Named here so it is
|
||||
a decision when it happens rather than a surprise.
|
||||
|
||||
**There is still no readiness-gated step.** Only an action carries `verify`; nothing declares that a
|
||||
service answers, so preparation that must happen *after* something is serving — seeding through its own
|
||||
API — remains unexpressible.
|
||||
|
||||
**Genesis keeps its own action.** At birth there is no control plane to derive anything, which is what
|
||||
[ADR 0067](0067-genesis-is-a-pivot.md) says about that moment.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **The composition carries the preparation, in the module's own context.** A test on a node's composed
|
||||
declaration: a version declaring a preparation entrypoint is preceded by it, and what it is given
|
||||
equals what the module's own code is given — asserted equal rather than written twice, which is the
|
||||
drift the superseded shape invited.
|
||||
- **A preparation that fails stops the version.** The host does not go past a step that did not complete,
|
||||
and the rollout stops at the first machine that did not take a version. Both are existing behaviours
|
||||
with existing tests; the test for preparation asserts the two together — the machine does not run it,
|
||||
and the machines after it are left alone.
|
||||
- **Once per version per state.** A test that a second convergence of the same version prepares nothing,
|
||||
and that a new version prepares again.
|
||||
- **The mesh's own control plane declares one.** The case that failed on 2026-09-28 is the case the tests
|
||||
cover, rather than a case a comment says is covered.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0133](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md) — what this supersedes, and why
|
||||
- [ADR 0052](0052-a-step-that-runs-once-before-a-container.md) — the host-side step that implements it for an image artifact
|
||||
- [ADR 0048](0048-a-provider-creates-the-credential-the-mesh-minted.md), [issue 022](../04-ISSUES/022-one-credential-per-node-per-provision-not-per-module/00-report.md) — a consumer is a module on a machine, which is what makes the scope derivable
|
||||
- [ADR 0018](0018-a-picture-is-read-from-what-runs.md) — a digest is the record that something happened
|
||||
- [ADR 0005](0005-the-node-host.md) — the control plane decides and never touches a machine
|
||||
- [ADR 0134](0134-the-mesh-says-what-it-applied.md) — what makes a failed preparation visible
|
||||
- [issue 133](../04-ISSUES/133-the-control-planes-schema-is-migrated-at-birth-and-never-again/00-report.md) — the failure that produced both records
|
||||
@@ -0,0 +1,106 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0052-a-step-that-runs-once-before-a-container.md
|
||||
---
|
||||
|
||||
# 136. A step gates its module, not the machine
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0052](0052-a-step-that-runs-once-before-a-container.md) made a run-once container a step the host
|
||||
runs to completion, and gave it the same reach a failed action has: it stops everything the declaration
|
||||
places after it. When the only steps on the mesh were a broker's seed and a forge's admin account, that
|
||||
reach was invisible — the thing after the step was the container the step existed for, in the same
|
||||
module.
|
||||
|
||||
[ADR 0135](0135-a-module-version-prepares-its-state-before-it-runs.md) made a step something the mesh
|
||||
derives for **any** module that prepares its state, and that turns the reach into a fault. A module
|
||||
whose database is briefly unreachable now stops every module declared after it on that machine, for as
|
||||
long as it is unreachable.
|
||||
|
||||
**The host already rejected this for every other shape, and says why in its own loop.** From
|
||||
[issue 011](../04-ISSUES/011-one-broken-module-blocks-every-other/00-report.md):
|
||||
|
||||
> It used to stop at the first one, and that made one broken resource hold the whole machine hostage: a
|
||||
> module declaring a package that does not exist meant every module ordered after it was never applied,
|
||||
> for ever, and the mesh reported "failed" without saying that the rest had not been tried. A machine
|
||||
> with one bad module and nine good ones ran none of the nine.
|
||||
|
||||
Everything is attempted and every failure reported — except an action and a run-once step, kept as the
|
||||
deliberate exceptions. So the mesh has two rules about the same question and the wider one is now
|
||||
reachable by any module that declares a schema.
|
||||
|
||||
**And it deadlocks a case the catalogue already named.** The catalogue migrates its own schema when it
|
||||
starts rather than in a step, and says why in its code: *a schema step that had to reach the provider
|
||||
over the overlay would block the very apply that brings the overlay up*. With a machine-wide gate that
|
||||
is exactly right — the step fails, the apply stops, the overlay module after it is never applied, and
|
||||
the next reconcile is blocked the same way. The module that most obviously wants a step could not have
|
||||
one.
|
||||
|
||||
## Decision
|
||||
|
||||
**A step gates its own module.** A run-once container that does not complete stops the rest of *that
|
||||
module's* resources and nothing else. Every other module on the machine is attempted, as every other
|
||||
shape already is.
|
||||
|
||||
**An action still gates the machine.** Genesis is a row of actions, each making the next possible, and
|
||||
they belong to no module — there is nothing narrower for their reach to be.
|
||||
|
||||
**What was not attempted is reported, not inferred from silence.** A skipped resource appears in the
|
||||
machine's account of the apply as skipped, with the reason, because "not attempted" and "nothing to do"
|
||||
are different answers and only one of them is somebody's to fix.
|
||||
|
||||
**A module is the part of a resource's identity before the first dot**, which is how the mesh composes
|
||||
them. What the mesh declares in its own right — a guard, an opening, the adoption's own resources —
|
||||
belongs to no module, and its gate is therefore the machine's.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **Leave the reach as it is.** Rejected: it reintroduces, through a mechanism now derived for every
|
||||
module, exactly the fault issue 011 removed. A mesh where one module's unreachable database stops a
|
||||
machine converging is worse than one where that module alone is behind.
|
||||
2. **Make preparation not a gate at all** — run it and carry on. Rejected: then a version serves against
|
||||
a state nobody shaped, which is the whole of what ADR 0135 exists to prevent.
|
||||
3. **Order every module's step before everything else on the machine**, so a gate stops nothing that
|
||||
matters. Rejected: it inverts the order a module needs — its files and directories are declared before
|
||||
its step because the step reads them — and it would still stop later modules.
|
||||
4. **Let a module declare how far its step reaches.** Rejected: the answer is the same for every module,
|
||||
and a field would let one be wrong about it.
|
||||
|
||||
## Consequences
|
||||
|
||||
**The catalogue can move to a step.** The reason it migrates at start — that a step blocks the apply
|
||||
that would make its provider reachable — stops being true: the step fails, that module waits, the
|
||||
overlay comes up, and the next reconcile prepares it. One shape for the whole mesh, which is what
|
||||
ADR 0135 asked for and could not have had.
|
||||
|
||||
**A module can sit behind while the machine is otherwise current.** That is the honest state and it is
|
||||
what the report now says. It also means a preparation that never succeeds is a module that never
|
||||
upgrades, quietly, until somebody reads the report — which is an argument for
|
||||
[ADR 0134](0134-the-mesh-says-what-it-applied.md) rather than against this.
|
||||
|
||||
**A module's resources must be ordered within the module for the gate to mean anything.** They already
|
||||
are: the mesh composes a module's resources in the order its manifest declares them, and its own
|
||||
workload comes after the files it reads.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **A failed step stops its module and nothing else.** A test with two modules: the one whose step
|
||||
failed does not start its workload, the other starts, and the error still says the failure gated
|
||||
something. It fails against the previous behaviour, which is how it was written.
|
||||
- **An action still stops the machine.** The existing test for a failed action is unchanged, and a step
|
||||
with no module in its identity — which is what genesis carries — takes the same path.
|
||||
- **The report names what was skipped.** Asserted in the same test, because a gate nobody can see is
|
||||
indistinguishable from a module that had nothing to do.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0052](0052-a-step-that-runs-once-before-a-container.md) — the step this narrows
|
||||
- [ADR 0135](0135-a-module-version-prepares-its-state-before-it-runs.md) — what made the reach reachable
|
||||
- [issue 011](../04-ISSUES/011-one-broken-module-blocks-every-other/00-report.md) — the same fault, removed once already
|
||||
- [ADR 0134](0134-the-mesh-says-what-it-applied.md) — how a module left behind becomes visible
|
||||
- mesh-host `internal/apply` — the loop whose own comment argued this case for every other shape
|
||||
@@ -0,0 +1,117 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: superseded
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
extends: 02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md
|
||||
reconstructed: false
|
||||
superseded-by: 02-DECISIONS/0140-the-filter-constrains-what-arrives-from-outside.md
|
||||
---
|
||||
|
||||
# 137. A machine says which networks it routes
|
||||
|
||||
## Context
|
||||
|
||||
The filter the mesh derives denies forwarding by default, because without a forward chain it says
|
||||
nothing about a container's published port
|
||||
([ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md), [issue 047](../04-ISSUES/047-the-firewall-does-not-cover-published-container-ports/00-report.md)).
|
||||
To keep a machine's own containers working it then allows two ranges: the container runtime's
|
||||
default bridge pool, and the pool its compose files are given. Those two are named in the
|
||||
controller's code, with a comment saying what the gap is:
|
||||
|
||||
> A machine whose runtime is configured with something else needs this to say so — which is a thing
|
||||
> the mesh cannot derive and a reason this list is named here rather than computed.
|
||||
|
||||
**There was no way to say so.** The list was a constant. A machine whose guests live anywhere else
|
||||
was filtered by a rule that looked deliberate and was a guess.
|
||||
|
||||
**Measured, on the day a workstation was converged.** Flipping it cut egress for five of its
|
||||
container networks at once, and for every network its test beds create — the beds allocate a fresh
|
||||
range per run, from a pool neither default covers. Nothing reported a fault. The containers could
|
||||
not reach anything, the machine went on reporting that it had applied what it was told, and the
|
||||
converge preview had said nothing about it either, because the preview lists what *listens* and
|
||||
routing is not a listener.
|
||||
|
||||
**And two questions, not one.** A guest also asks its host for an address and for names. Both arrive
|
||||
at the input chain, where nothing declared them, so denying by default left the guests of a routed
|
||||
network with no address and no resolution — which is not a closed port but a network that does not
|
||||
function, asked for by this machine's own guest.
|
||||
|
||||
**Why the machine cannot simply be read.** A test bed creates its bridge while it runs, between one
|
||||
declaration and the next, so a filter derived from what the machine last reported would be correct
|
||||
only for the networks that already existed when it was composed. A declared range covers the ones
|
||||
that do not exist yet.
|
||||
|
||||
## Decision
|
||||
|
||||
**A machine says which networks it routes for what it hosts, and the filter forwards them.** A
|
||||
node-level fact, beside the node's public domain
|
||||
([ADR 0066](0066-public-routing-is-name-agnostic.md)) and for the same reason: the
|
||||
machine routes them, and the module that loads the filter holds a seat and may be replaced.
|
||||
|
||||
**Added to the runtime's defaults, never replacing them.** A machine that names one range has not
|
||||
stopped hosting whatever was already on the runtime's own pools, and replacing would trade one
|
||||
silent breakage for another.
|
||||
|
||||
**Their guests keep address and name service.** For a network that was named, the input chain admits
|
||||
that network's own DHCP and DNS, and nothing else: everything else a guest might want from its host
|
||||
is a port somebody declares, like every other port on this machine.
|
||||
|
||||
**Said in CIDR form and checked when it is said.** An entry that does not parse is a line nftables
|
||||
refuses, and a refused ruleset is a machine filtering nothing while its unit reports a fault — so
|
||||
the refusal happens where a person can read it, not on the machine.
|
||||
|
||||
**A machine that says nothing is filtered exactly as before.** Every machine already converged is
|
||||
untouched by this.
|
||||
|
||||
## Options considered
|
||||
|
||||
1. **Leave it constant and edit the code per installation.** Rejected: the value is a property of
|
||||
one machine, the code is the whole mesh's, and the two ranges as they stand describe a machine
|
||||
whose runtime was left at its defaults. It is also how this got here.
|
||||
2. **Derive it from what the machine reports.** Rejected as insufficient, not as wrong: it cannot
|
||||
cover a network created between two declarations, which is precisely the case that was broken. It
|
||||
would also make the filter follow whatever appeared on the machine, which is a firewall that
|
||||
widens itself.
|
||||
3. **A per-node setting on the module that loads the filter.** Rejected: the machine routes the
|
||||
networks. The filter module holds a node-scoped seat and is meant to be replaceable, and a
|
||||
replacement must not lose the machine's own truth.
|
||||
4. **Replace the defaults with what is said.** Rejected: see the decision. The first machine to name
|
||||
its bed range would lose its containers.
|
||||
5. **Admit all input from a routed network, not only address and name service.** Rejected: that is
|
||||
every port on the machine open to anything it hosts, which is the derivation abandoned.
|
||||
|
||||
## Consequences
|
||||
|
||||
**The converge preview says what a machine routes**, including when it routes nothing but the
|
||||
defaults, with the command that changes it. The preview's own sentence about traffic it cannot
|
||||
preview stays, because a tunnel and the found firewall's NAT are still not previewable.
|
||||
|
||||
**A machine whose guests are already broken by an earlier flip is fixed by saying its networks and
|
||||
pushing**, with no flip to undo.
|
||||
|
||||
**The list is one more thing that can be wrong and stale.** A range removed from the machine and
|
||||
left here keeps forwarding for a network that no longer exists, which admits nothing, because there
|
||||
is no guest on it to admit. That is the safe direction of being out of date.
|
||||
|
||||
## How this is checked
|
||||
|
||||
- **What a machine says it routes is forwarded, and its guests keep address and name service.** A
|
||||
test renders a ruleset for a machine that names one range and asserts both chains, per chain body
|
||||
so a line in the wrong chain cannot pass it. It fails against the previous behaviour, which is how
|
||||
it was written.
|
||||
- **The runtime's own defaults survive naming a range.** Asserted in the same test.
|
||||
- **A machine that names nothing renders byte-identically to one that names nil**, so every machine
|
||||
already behind this filter is untouched.
|
||||
- **Each family is matched in its own syntax.** A test with one v4 and one v6 network asserts
|
||||
`ip saddr` and `ip6 saddr`, because one set holding both is a syntax error and a ruleset that does
|
||||
not load is a machine filtering nothing.
|
||||
- **An entry that is not a network is refused where it is said**, by the parse in the setter.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md) — the derived filter this completes
|
||||
- [ADR 0066](0066-public-routing-is-name-agnostic.md) — the precedent for a node-level fact
|
||||
- [issue 047](../04-ISSUES/047-the-firewall-does-not-cover-published-container-ports/00-report.md) — why there is a forward chain at all
|
||||
- [issue 137](../04-ISSUES/137-converging-a-machine-cut-off-its-own-guests/00-report.md) — the measurement that produced this
|
||||
- mesh-controller `internal/catalogue/filtering.go` — the constant whose own comment named this gap
|
||||
@@ -0,0 +1,154 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md
|
||||
---
|
||||
|
||||
# 138. An assignment binds an endpoint and says how far it reaches
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md) settled that a machine's
|
||||
packet filter is derived from what its modules declare they listen on, and that the `from` of a
|
||||
listen "is the whole of public-versus-internal". That was true of the packet filter, and it turned
|
||||
out to be true of nothing else.
|
||||
|
||||
Reachability is now settled three times, in three places, by three mechanisms that cannot disagree
|
||||
out loud ([issue 140](../04-ISSUES/140-an-endpoints-reach-is-not-declared/00-report.md)):
|
||||
|
||||
- **The filter** reads a listen's source, and a per-node setting may override it. That setting has
|
||||
exactly one caller in the control plane — the function that builds the node's rules.
|
||||
- **The names** come from a route contribution, which names a label and a port and says nothing
|
||||
about reach. The reverse proxy composes a **public** name and an **internal** name for every route
|
||||
it is given, because it can.
|
||||
- **The certificate authority** follows from which names exist. Measured on the control-node: an
|
||||
identity provider carries a public certificate valid 90 days and an internal one valid 24 hours and
|
||||
renewed daily. No assignment asked for either.
|
||||
|
||||
So *this endpoint must not be public* cannot be written. It is therefore enforced by nothing, while a
|
||||
public certificate for that very name is obtained automatically — the fault
|
||||
[how-we-build.md](../00-META/how-we-build.md) names, an unenforced rule being indistinguishable from
|
||||
a wrong one, with the additional cost that the wrong thing is done eagerly.
|
||||
|
||||
And a port that is not routed cannot be spoken about at all beyond the filter. The forge serves git
|
||||
over ssh; that endpoint has no name, no certificate and no way to be called public except a key only
|
||||
the filter reads.
|
||||
|
||||
**Two per-node settings already exist and are half of this.** One gives a module's declared port a
|
||||
machine port. One overrides a declared port's source. They key on port numbers, so nothing ties a
|
||||
port to the route that serves it: a route contribution names a port too, and the two are equal only
|
||||
by coincidence.
|
||||
|
||||
**Where this belongs is already decided.** [ADR 0046](0046-a-module-configuration-is-its-assignments-not-its-manifest.md)
|
||||
says a module's configuration is its assignments. Whether the forge answers git-over-ssh from the
|
||||
public internet is a fact about one installation and one machine, not a property of the software —
|
||||
and [ADR 0112](0112-a-module-definition-names-no-node-mesh-or-path.md) already refuses an
|
||||
installation's decisions in a definition.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Leave reach in the manifest, as `from` today.** Rejected: it is an installation's decision
|
||||
written into the definition, and it cannot differ between two machines running the same module —
|
||||
which is exactly the case the forge presents.
|
||||
2. **Extend the existing source override to the names and the certificate, without naming
|
||||
endpoints.** Rejected: it keys on a port number. A module's route contribution names a port as
|
||||
well, and nothing says the two are the same thing, so one statement cannot be made to reach all
|
||||
three mechanisms. Naming the endpoint is what makes that possible.
|
||||
3. **Derive reach from whether the node has a public domain recorded.** Rejected: that is a property
|
||||
of the machine, and two endpoints on one machine differ — a database and a web front end on the
|
||||
same host.
|
||||
4. **A fourth reach for "public name, internal authority"** — a name that resolves publicly and must
|
||||
not appear in a public issuance log, obtained by DNS-01. Deferred, not rejected: it is a real case
|
||||
and it is a question about which challenge an authority uses, not about how far an endpoint
|
||||
reaches. Left to the certificate work as an open question.
|
||||
5. **Make the manifest silent on reach and require every assignment to state it.** Rejected for the
|
||||
transition: every endpoint reachable today would close until an assignment named it, which is a
|
||||
flag day across the whole catalogue.
|
||||
|
||||
## Decision
|
||||
|
||||
**A module declares named endpoints.** An endpoint is one port the module serves, with a name the
|
||||
module chooses, its protocol, and what it is for. A route contribution **names the endpoint it
|
||||
routes** rather than repeating a port number. The manifest says what the module serves and what it
|
||||
would serve it to by default; it does not say what this installation does with it.
|
||||
|
||||
**An assignment binds each endpoint and says how far it reaches.** Per node: the machine port the
|
||||
endpoint is published on, and its **reach** — one of `internal`, `public` or `both`. An assignment
|
||||
that states nothing keeps the manifest's default, so no machine changes until an assignment says so.
|
||||
|
||||
**Reach means all three mechanisms at once, and is the only thing that decides them.**
|
||||
|
||||
- `internal` — the filter opens the machine port to the private network; the proxy serves the
|
||||
internal name and not the public one; the certificate comes from the mesh's own authority.
|
||||
- `public` — the filter opens it to anywhere; the proxy serves the public name; the certificate
|
||||
comes from the public authority.
|
||||
- `both` — both names, each from its own authority, and the filter opens to anywhere.
|
||||
|
||||
**An endpoint that is not routed is reached but never named.** An endpoint with no route contribution
|
||||
yields filter rules and nothing else: no name is composed and no certificate is requested. Git over
|
||||
ssh is that case, and it is the case the model could not express.
|
||||
|
||||
**The authority stops being chosen by which names happen to exist.** The proxy composes the names the
|
||||
assignments asked for, and asks each name's own authority for it. A name nobody asked for is not
|
||||
composed, so it is not certified.
|
||||
|
||||
**The two existing settings are this, completed.** The per-node port mapping becomes the endpoint's
|
||||
binding. The per-node source override becomes its reach, widened from the filter alone to the names
|
||||
and the certificate as well.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **A manifest gains endpoint names, and a route contribution names an endpoint instead of a port.**
|
||||
Every routed module's manifest changes. The word ships one release before any manifest uses it, and
|
||||
reaches the build machine and the control plane first.
|
||||
- **One derived value is read by three things** — the filter's rules, the proxy's contributions, the
|
||||
certificate request — so they can no longer disagree, and a disagreement becomes a refusal at the
|
||||
assignment rather than a surprise on a machine.
|
||||
- **A name that must not be public becomes writable, and therefore checkable.** It also gives
|
||||
[issue 129](../04-ISSUES/129-nothing-makes-a-machine-trust-the-meshs-authority/00-report.md) a
|
||||
declared answer to read: which endpoints are internal is what says whose root must be installed
|
||||
where.
|
||||
- **[Issue 139](../04-ISSUES/139-an-internal-route-name-resolves-to-the-consumers-node/00-report.md)
|
||||
becomes answerable**: the endpoint's assignment names the machine that serves it, which is the fact
|
||||
the internal name should be composed from.
|
||||
- **Reach becomes reportable.** The mesh can say, per endpoint, where it is reachable from and which
|
||||
authority holds its certificate — neither of which `status` can say today.
|
||||
- **This narrows [ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md).** Its
|
||||
decision stands: the firewall is derived and host-applied, not a provider. What no longer holds is
|
||||
that a listen's `from` is the whole of public-versus-internal; it is the filter's share of a
|
||||
statement that also governs names and certificates.
|
||||
- **What got harder:** every endpoint needs a name, including a module that serves exactly one port
|
||||
and had no reason to name it. And an installation that wants a module public must now say so on the
|
||||
assignment rather than inheriting it from the definition, which is more to say and the reason it is
|
||||
right.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **One module, two endpoints, different reach.** A module declaring an internal endpoint and a
|
||||
public one renders a filter opening one to the private network and one to anywhere, asserted per
|
||||
chain body so a rule in the wrong chain cannot pass.
|
||||
- **The names follow the reach.** The same module's routed endpoint composes the internal name only
|
||||
when internal, the public name only when public, and both when both — and a certificate is
|
||||
requested from the matching authority for each name composed and for no other. This fails against
|
||||
the previous behaviour, where both names and both certificates are always composed, which is how
|
||||
it is written.
|
||||
- **An unrouted endpoint is filtered and never named.** Asserted for an endpoint with reach and no
|
||||
route contribution: rules rendered, no contribution, no certificate request.
|
||||
- **An assignment naming an endpoint the module does not declare is refused where it is said**, as is
|
||||
a reach that is not one of the three — before it reaches a machine, because a ruleset that does not
|
||||
load is a machine filtering nothing.
|
||||
- **An assignment that states nothing renders byte-identically to today**, so every machine already
|
||||
converged is untouched until its assignment says otherwise.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md) — narrowed here
|
||||
- [ADR 0046](0046-a-module-configuration-is-its-assignments-not-its-manifest.md) — where reach belongs
|
||||
- [ADR 0066](0066-public-routing-is-name-agnostic.md) — the public name this composes
|
||||
- [ADR 0112](0112-a-module-definition-names-no-node-mesh-or-path.md) — why reach is not a definition's
|
||||
- [issue 140](../04-ISSUES/140-an-endpoints-reach-is-not-declared/00-report.md) — the measurement
|
||||
- [issue 139](../04-ISSUES/139-an-internal-route-name-resolves-to-the-consumers-node/00-report.md),
|
||||
[issue 129](../04-ISSUES/129-nothing-makes-a-machine-trust-the-meshs-authority/00-report.md)
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: superseded
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0137-a-machine-says-which-networks-it-routes.md
|
||||
superseded-by: 02-DECISIONS/0140-the-filter-constrains-what-arrives-from-outside.md
|
||||
---
|
||||
|
||||
# 139. A network is forwarded because a module declared it
|
||||
|
||||
## Context
|
||||
|
||||
[ADR 0137](0137-a-machine-says-which-networks-it-routes.md), decided the same week, gave a machine a
|
||||
way to say which networks it routes for its guests. It was written because the derived filter's
|
||||
forward chain allowed two ranges named as constants in the control plane's source — the container
|
||||
runtime's bridge pool, and part of the pool its compose files are given — with a comment admitting
|
||||
the gap: *a machine whose runtime is configured with something else needs this to say so, which is a
|
||||
thing the mesh cannot derive.*
|
||||
|
||||
**It can be derived, and from the right place.** Measured on the last machine still to be converged
|
||||
([issue 141](../04-ISSUES/141-the-forward-chain-does-not-follow-the-modules/00-report.md)): twenty-one
|
||||
container networks, nine inside the runtime's bridge pool, twelve in the other private range, and six
|
||||
of those outside the constant's lower bound — so the flip would have cut their guests off exactly as
|
||||
it did on the workstation that produced 0137.
|
||||
|
||||
Naming a range to cover the six is what 0137 provides for, and it is the wrong instrument. Of those
|
||||
six networks, **four are networks the mesh's own modules declare**, present as network resources in
|
||||
the node's plan and created by the host because a module asked for them. **Two are the predecessor's
|
||||
leftovers** — compose networks of services the mesh does not run. Any range wide enough to keep the
|
||||
four forwards the two as well: a firewall widened by hand to protect networks that should not exist.
|
||||
|
||||
The mesh already knows which of the twenty-one are its own, because it made them.
|
||||
|
||||
**And the node's configuration is meant to follow the modules assigned to it.** That is the mesh's
|
||||
founding shape — the machine runs modules, and its files, its filter and its accounts are composed
|
||||
from what runs there ([ADR 0005](0005-the-node-host.md),
|
||||
[ADR 0010](0010-delivery.md),
|
||||
[ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md)). The forward chain is the
|
||||
one derived thing that consults a constant and a list a person types.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Keep 0137 as it stands** — two constants plus a named list. Rejected: the list is written in
|
||||
addresses, and addresses are what the runtime allocates, so the only entry safe enough to keep a
|
||||
machine working is wider than the truth. It cannot distinguish a network the mesh made from one
|
||||
left behind, which is the distinction that decides whether forwarding it is correct.
|
||||
2. **Derive it from what the machine reports.** Still rejected, on 0137's own grounds: a test bed
|
||||
creates its bridge between one declaration and the next, and a filter that follows whatever
|
||||
appeared on a machine is a firewall that widens itself. **This decision is not that** — see below.
|
||||
3. **Have the control plane allocate each module network's range from a pool it owns,** so it can
|
||||
render the address itself. Rejected: more machinery for no gain. The runtime already allocates and
|
||||
the host already knows, and taking allocation over means the mesh owning an address space it has no
|
||||
other reason to own.
|
||||
4. **Have each module declare its network's range.** Rejected by
|
||||
[ADR 0112](0112-a-module-definition-names-no-node-mesh-or-path.md): a definition names no address,
|
||||
and the same definition runs on machines whose runtimes have allocated differently.
|
||||
|
||||
## Decision
|
||||
|
||||
**A network is forwarded because a module declared it.** Per node, the forward chain forwards the
|
||||
networks of the modules assigned there, and by default nothing else. A module unassigned stops being
|
||||
forwarded at the next reconcile.
|
||||
|
||||
**The host resolves a declared network to its addresses.** A network resource carries a name; the
|
||||
runtime allocates the subnet when the network is created. So the control plane declares *forward the
|
||||
networks these modules asked for* and the host — which made them, and already resolves a container by
|
||||
its name — renders the addresses. [ADR 0005](0005-the-node-host.md) holds: the host applies, it does
|
||||
not decide.
|
||||
|
||||
**Deriving from the declaration is not deriving from the machine.** Both of 0137's objections fall
|
||||
away. The set is known before the network exists, because a module declared it, so a network created
|
||||
between two declarations is already in the one that asked for it. And it cannot widen itself: a
|
||||
network nobody declared is never forwarded, however it appeared on the machine.
|
||||
|
||||
**The runtime's own default bridge is forwarded, from what the runtime reports.** Containers that name
|
||||
no module network attach to it, and it belongs to the runtime rather than to any module — so the host
|
||||
renders it from what the runtime says, not from a range named in the control plane. The constants go.
|
||||
|
||||
**What a machine says is for guests no module declares.** A test bed is not a module and its range is
|
||||
not a module's; that is the case 0137's mechanism is for, and it keeps it — added to the derived set,
|
||||
never replacing it, as 0137 decided. Narrowed to that, it is named for it.
|
||||
|
||||
**Their guests keep address and name service**, per declared network, unchanged from
|
||||
[ADR 0137](0137-a-machine-says-which-networks-it-routes.md): the input chain admits that network's own
|
||||
DHCP and DNS and nothing else.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **The two constants are removed**, and with them the class of fault that a machine's guests depend
|
||||
on a range that describes some other machine.
|
||||
- **This is a behaviour change, not a refactor.** On the machine measured, the derived set and the
|
||||
constant do not cover the same ground — that is the whole reason for the record. A machine whose
|
||||
module networks happen to fall inside the old ranges renders the same rules.
|
||||
- **A range that exists only to keep a leftover alive becomes visible as such**, because it will not
|
||||
be in the derived set and has to be said out loud to survive.
|
||||
- **`node networks` narrows** to guests no module declares, and the preview says which of a machine's
|
||||
networks are the mesh's and which are not, so the difference is readable before a flip rather than
|
||||
after.
|
||||
- **A module's declaration gains nothing.** It already declares its network; what changes is that the
|
||||
filter reads it.
|
||||
- **What got harder:** the host renders part of the forward chain from what it created, so the
|
||||
control plane no longer holds the whole rule set as text. The rule the mesh states is the set of
|
||||
networks; the addresses are the machine's.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **Only declared networks are forwarded.** A node with two modules that declare networks renders
|
||||
forward rules for exactly those two, and none for a third network present on the machine that no
|
||||
module declared. This fails against the previous behaviour, which forwards by range and cannot tell
|
||||
them apart, and that is how it is written.
|
||||
- **Unassigning a module removes its network's rule** at the next reconcile, asserted on the rendered
|
||||
chain rather than on the intent.
|
||||
- **Guests of a declared network keep address and name service**, asserted per chain body so a line in
|
||||
the wrong chain cannot pass — carried from 0137.
|
||||
- **The runtime's own default bridge comes from the runtime**, asserted by rendering for a runtime
|
||||
whose default bridge is somewhere other than the range the constant named.
|
||||
- **A machine that names a range for guests no module declares still gets it**, added to the derived
|
||||
set and not replacing it.
|
||||
- **Each family is matched in its own syntax**, carried from 0137: one set holding both is a syntax
|
||||
error, and a ruleset that does not load is a machine filtering nothing while its unit reports a
|
||||
fault.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0137](0137-a-machine-says-which-networks-it-routes.md) — narrowed here; its mechanism keeps the
|
||||
case it is right for
|
||||
- [ADR 0005](0005-the-node-host.md) — the host applies; the addresses are the machine's
|
||||
- [ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md) — the filter is derived from
|
||||
what runs there
|
||||
- [ADR 0112](0112-a-module-definition-names-no-node-mesh-or-path.md) — why a module does not name its
|
||||
range
|
||||
- [issue 141](../04-ISSUES/141-the-forward-chain-does-not-follow-the-modules/00-report.md) — the
|
||||
measurement
|
||||
- [issue 137](../04-ISSUES/137-converging-a-machine-cut-off-its-own-guests/00-report.md) — the
|
||||
breakage that produced 0137
|
||||
@@ -0,0 +1,146 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-28
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md
|
||||
supersedes:
|
||||
- 02-DECISIONS/0137-a-machine-says-which-networks-it-routes.md
|
||||
- 02-DECISIONS/0139-a-network-is-forwarded-because-a-module-declared-it.md
|
||||
---
|
||||
|
||||
# 140. The filter constrains what arrives from outside, and says nothing about a machine's own guests
|
||||
|
||||
## Context
|
||||
|
||||
The filter the mesh derives blocks traffic passing *through* a machine unless something allows it,
|
||||
because a container's published port is traffic passing through rather than traffic arriving at the
|
||||
machine itself ([ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md),
|
||||
[issue 047](../04-ISSUES/047-the-firewall-does-not-cover-published-container-ports/00-report.md)).
|
||||
Having blocked all of it, the filter then had to let the machine's own containers reach outward again.
|
||||
It does that by listing the address ranges those containers sit on.
|
||||
|
||||
As rendered on a converged workstation today:
|
||||
|
||||
```
|
||||
policy drop
|
||||
ct state established,related accept
|
||||
ip saddr 172.16.0.0/12 accept
|
||||
ip saddr 192.168.128.0/17 accept
|
||||
ip saddr 10.0.0.0/8 accept
|
||||
ip saddr 192.168.16.0/20 accept
|
||||
... four more
|
||||
```
|
||||
|
||||
Two of those ranges were constants in the control plane's source. The rest were typed by the operator
|
||||
after [ADR 0137](0137-a-machine-says-which-networks-it-routes.md), which existed to make the typing
|
||||
possible, because converging that workstation had cut every one of its containers off from the
|
||||
internet and nothing reported a fault
|
||||
([issue 137](../04-ISSUES/137-converging-a-machine-cut-off-its-own-guests/00-report.md)).
|
||||
|
||||
**The list is the mistake, not its contents.** Every attempt to make it correct fails the same way.
|
||||
A constant describes one machine. A typed range goes stale, and cannot tell a network the mesh made
|
||||
from one a predecessor left behind — measured on the control-node, where six such ranges fall outside
|
||||
the constants and two of the six belong to services the mesh does not run
|
||||
([issue 141](../04-ISSUES/141-the-forward-chain-does-not-follow-the-modules/00-report.md)).
|
||||
[ADR 0139](0139-a-network-is-forwarded-because-a-module-declared-it.md) tried to generate the same
|
||||
list from the modules and put half the rule set on the machine to do it. Three records, one list, and
|
||||
the list should not exist.
|
||||
|
||||
**Because the mesh has no policy about a container reaching outward.** What the filter is for is
|
||||
stated in [ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md): which port is open,
|
||||
and to whom. That is about what arrives. A container of this machine's own opening a connection to
|
||||
something else is not a port being opened to anybody, and enumerating the addresses it might do so
|
||||
from is bookkeeping about the machine's internal plumbing, which the mesh neither owns nor can know.
|
||||
|
||||
**The system being replaced never had this fault, and its rule says why.** The chain still protecting
|
||||
the control-node applies only to traffic arriving on that machine's outward link, and leaves
|
||||
everything else alone. The mesh's filter dropped that distinction and replaced it with a list of
|
||||
addresses.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Keep the list and generate it better** — from the modules' declared networks, or from what the
|
||||
machine reports. Rejected: [ADR 0139](0139-a-network-is-forwarded-because-a-module-declared-it.md)
|
||||
is that, and it puts part of the rule set on the machine, which makes the rule set partly the
|
||||
machine's and the derivation advisory.
|
||||
2. **Name the guest links instead of their addresses, and allow only those.** Rejected as more than is
|
||||
needed: it fails in the safe direction, but it is still a list that has to keep up with the
|
||||
machine, and the thing it protects against — a container reaching outward — is not a thing the mesh
|
||||
has a position on.
|
||||
3. **Do not block traffic passing through at all.** Rejected: that is
|
||||
[issue 047](../04-ISSUES/047-the-firewall-does-not-cover-published-container-ports/00-report.md),
|
||||
where a published port was reachable from anywhere because no rule mentioned it.
|
||||
4. **Constrain what arrives from outside, and nothing else.** Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**The filter constrains traffic arriving from outside the machine, and says nothing about traffic that
|
||||
did not.** Traffic passing through the machine is allowed unless it arrived on one of the machine's
|
||||
outward links, in which case it is allowed only where a declared endpoint's reach admits it
|
||||
([ADR 0138](0138-an-assignment-binds-an-endpoint-and-says-how-far-it-reaches.md)). A container of this
|
||||
machine's own reaching anywhere is not filtered, because the mesh has no position on it.
|
||||
|
||||
**A machine says which of its links face outside.** One node-level fact, reported by the machine the
|
||||
way it already reports the kind of firewall it found and the tunnel it carries — not a setting, not a
|
||||
list of addresses, and not something anybody types. It does not change when a module is added or
|
||||
removed, which is what separates it from the list it replaces.
|
||||
|
||||
**A machine that has reported no outward link is sent no filter.** Rendering a rule around a link
|
||||
whose name is not known produces a rule set that does not load, which is a machine filtering nothing
|
||||
while its unit reports success. The refusal happens in the control plane, where a person reads it, and
|
||||
the machine keeps the filter it already has.
|
||||
|
||||
**No addresses of the machine's own networks appear in the filter.** The two constants are removed and
|
||||
`node networks` is removed with them, along with everything any machine was told to say through it.
|
||||
Ports continue to follow the modules exactly as before: a module assigned to a machine opens the port
|
||||
its assignment says it reaches on, and nothing about a network is said anywhere.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **Three records collapse into one rule.** 0137 and 0139 are superseded. What 0137 was right about —
|
||||
that converging a machine had silently cut off its own containers, and that nothing previewed it — is
|
||||
answered by removing the cause rather than by giving the operator a way to compensate for it.
|
||||
- **Every machine already converged loses its declared ranges and keeps working**, because the traffic
|
||||
those ranges allowed is now allowed by not having arrived from outside. The workstation's five ranges
|
||||
and the laptop's one are deleted rather than migrated.
|
||||
- **A machine's test beds stop being a special case.** A bed's network is created while the machine
|
||||
runs and was the case no list could cover; it is now covered by not being mentioned.
|
||||
- **A new fact travels in the report**, and the control plane refuses to compose a filter without it,
|
||||
so the order of the roll-out matters: the machines report before the control plane depends on it.
|
||||
- **A machine with more than one outward link says so**, and a machine that acquires one while the mesh
|
||||
is not looking is treated as internal until its next report. That window is the cost of this shape;
|
||||
it is bounded by the report interval, and it exists on machines whose outward link changes, which
|
||||
are the machines with nothing published to the outside.
|
||||
- **What got harder:** nothing in the declaration, and one more thing a machine must be able to work
|
||||
out about itself. A machine that cannot say which link faces outside cannot be given a filter.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A machine's own container reaches outward with no network named anywhere.** A bed converges a
|
||||
machine carrying containers on several networks, none of them mentioned in any setting, and each
|
||||
reaches out afterwards. This fails against the previous behaviour, where the same flip cut them off,
|
||||
and that is how it is written.
|
||||
- **A port declared reachable from outside is reachable; one that is not, is not.** Probed from off the
|
||||
machine's private network, for a published port and for an undeclared one, before and after the flip.
|
||||
- **A network created after the filter was composed needs no new filter.** A network is made on the
|
||||
machine after its last declaration and a container on it reaches out, with nothing re-sent.
|
||||
- **No address of a machine's own networks appears in a rendered filter**, asserted on the text so a
|
||||
range cannot creep back in.
|
||||
- **A machine that reports no outward link is sent no filter, and the refusal names it** — asserted in
|
||||
the control plane, and that the machine's existing filter is left alone.
|
||||
- **A machine reporting two outward links has both constrained**, asserted per chain body so a rule
|
||||
covering one and not the other cannot pass.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0045](0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md) — what the filter is for
|
||||
- [ADR 0138](0138-an-assignment-binds-an-endpoint-and-says-how-far-it-reaches.md) — what admits traffic
|
||||
arriving from outside
|
||||
- [ADR 0100](0100-a-node-in-use-is-adopted-before-it-is-converged.md) — why traffic passing through is
|
||||
filtered at all
|
||||
- [ADR 0137](0137-a-machine-says-which-networks-it-routes.md),
|
||||
[ADR 0139](0139-a-network-is-forwarded-because-a-module-declared-it.md) — superseded here
|
||||
- [issue 137](../04-ISSUES/137-converging-a-machine-cut-off-its-own-guests/00-report.md),
|
||||
[issue 141](../04-ISSUES/141-the-forward-chain-does-not-follow-the-modules/00-report.md)
|
||||
@@ -0,0 +1,154 @@
|
||||
---
|
||||
topic: what runs on it
|
||||
status: accepted
|
||||
date: 2026-09-29
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0005-the-node-host.md
|
||||
---
|
||||
|
||||
# 141. The host delivers its own successor, and versions live side by side
|
||||
|
||||
## Context
|
||||
|
||||
[Issue 142](../04-ISSUES/142-the-host-is-the-one-thing-the-mesh-does-not-deliver/00-report.md). A
|
||||
merge builds every changed module and the control plane — which is itself a module — and the result
|
||||
reaches the machines running it with nobody asking. The host is the exception: it is not a build
|
||||
target, no declaration delivers it, and every machine in this mesh runs a byte-identical binary that
|
||||
somebody built on a workstation and copied out.
|
||||
|
||||
The half that *recovers* from a bad host exists. `internal/upgrade` can tell that the executable this
|
||||
process started from was replaced on disk, and it records which version last completed a reconcile.
|
||||
The launcher counts consecutive failed starts, calls a rollback at the limit, and treats a clean exit
|
||||
as the host standing aside so that the next loop runs whatever is on disk now. That supervision is
|
||||
complete and correct.
|
||||
|
||||
Two things make it dead code:
|
||||
|
||||
- **`Replaced()` is called by nothing but its own tests.** Nothing tells the running host that a
|
||||
successor is waiting.
|
||||
- **The rollback resolves a version through the machine's package manager** — `pacman -U` from the
|
||||
package cache. No machine here has the host installed as a package, so the recovery cannot run on
|
||||
any of them; and being written in one package manager's terms, it cannot run on two of the three
|
||||
operating systems the host is built for — [ADR 0005](0005-the-node-host.md) builds one binary per
|
||||
operating system, pinned at link time.
|
||||
|
||||
**The record already points at the answer.** What is kept is a *version*, not a path. Keeping a
|
||||
version is only useful to something that can choose between versions present on the machine, which is
|
||||
what the package manager was being asked to do. The versions can simply be on disk.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Deliver the host as a package, as the rollback assumes.** Rejected: it needs a package built and
|
||||
a repository trusted per operating system, three of each, and the existing `package` resource
|
||||
asserts presence and deliberately never a version — "version is the package manager's business and
|
||||
the mesh does not hold a second opinion about it" — so it cannot ask for a particular host anyway.
|
||||
Heaviest of the three and the only one that is different on every machine.
|
||||
2. **Write the new binary over the running one.** Rejected on a fact: a running executable cannot be
|
||||
truncated, and `archive` opens what it unpacks with `O_TRUNC`. It could be made to write and
|
||||
rename, which is better hygiene and worth doing for its own sake, but it buys nothing here that
|
||||
option 3 does not, and it leaves rollback with nowhere to go back to.
|
||||
3. **Versions side by side; the newest retires the old.** Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**A host version is delivered as an archive into a directory named for it, and never over a running
|
||||
one.** The declaration names it like any other archive — fetched by digest, the digest checked before
|
||||
anything is unpacked. Nothing new travels, no new resource kind, and no change to how archives are
|
||||
applied, because the path being written is not the path being executed.
|
||||
|
||||
**The launcher starts the most recently delivered version.** That is what "the newest" means: the
|
||||
version whose directory arrived last. It reads no pointer and follows no link — the mesh creates no
|
||||
links ([ADR 0012](0012-the-mesh-creates-no-symlinks.md)) — and the version is in the path, so nothing
|
||||
has to be told what is running.
|
||||
|
||||
**The running host stands aside for a successor, and only between reconciles.** Finding a newer
|
||||
version delivered, it finishes the reconcile it is in and exits cleanly. The launcher already reads a
|
||||
clean exit as exactly this and starts what is on disk now. A host that stood aside mid-apply is the
|
||||
half-configured machine this project exists to prevent, so the check happens at the boundary and
|
||||
nowhere else.
|
||||
|
||||
**A version that completes a reconcile records itself, and retires what came before it.** The
|
||||
known-good record is written as it is today. Then versions older than the one before the running one
|
||||
are removed: the running version and its predecessor are kept, which is exactly what a rollback
|
||||
needs, and nothing else accumulates.
|
||||
|
||||
**Rollback starts the previous version instead of reinstalling a package.** At the failure limit the
|
||||
launcher pins the known-good version and starts that, once. The second failure is still a different
|
||||
diagnosis — the previously working version does not run either, so it is the machine and not the
|
||||
binary — and the halt is unchanged. No package manager, no package cache, and the same script on every
|
||||
operating system.
|
||||
|
||||
**A machine says which host version it is running,** on the report it already sends, beside the other
|
||||
facts it states about itself. Without it nothing can say a machine is behind, so "every machine
|
||||
current with its source" cannot include the host.
|
||||
|
||||
## Progressive insight — 2026-09-29, the same day
|
||||
|
||||
**The delivery is not "nothing new", and this record said it was.** The decision above stands and is
|
||||
built: versions side by side, the newest runs, the running host stands aside between reconciles, a
|
||||
completed reconcile retires what is older than the predecessor, rollback picks a directory. What was
|
||||
wrong was a claim about how a version reaches a machine. The paragraph on delivery said the
|
||||
declaration "names it like any other archive… nothing new travels, no new resource kind"; the second
|
||||
half is true and the first is not, because two things the delivery needs do not exist:
|
||||
|
||||
- **Nothing can compile it.** A `bundle` artifact is compiled by a closed list of toolchains —
|
||||
typescript and python — whose own comment says adding a language is a decision, because a language
|
||||
used by *modules* needs an SDK carrying the broker client, the event envelope and tool serving. The
|
||||
host uses none of that: it is what applies modules, not one of them. So the obligation that list
|
||||
warns about attaches to a module written in a language, not to the language being buildable, and
|
||||
the control plane — also written in Go — is built as an image from a Dockerfile rather than through
|
||||
a toolchain at all.
|
||||
- **A version cannot reach the path.** An `archive` resource names a fixed path in the manifest, and
|
||||
nothing interpolates the built version into it, so nothing can ask for
|
||||
`…/versions/<version>/`.
|
||||
|
||||
Neither changes what was decided, which options were weighed, or any consequence: the shape is
|
||||
unaffected and the host half is merged and tested. What it changes is the cost, which this record
|
||||
understated as none. The remaining work is a way to build the host and a way to name a version in a
|
||||
path, and until both exist nothing delivers a version and every machine takes the fallback — which is
|
||||
what every machine does today.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **The host becomes a build target and a module** — a module whose resource is the next host, applied
|
||||
by the host that is running. The bootstrap is not circular because the two are different versions in
|
||||
different directories.
|
||||
- **Rollback becomes usable on every machine**, having been usable on none. It also stops being
|
||||
written in one operating system's terms.
|
||||
- **One copy by hand remains, once.** The first host that understands versioned directories cannot be
|
||||
fetched by a host that does not. That copy is the last, and it is the honest cost of the change
|
||||
rather than a step in the design.
|
||||
- **Two versions occupy disk instead of one.** About nine megabytes. The predecessor is the price of a
|
||||
rollback that does not depend on a cache somebody else may clean.
|
||||
- **What got harder:** a host must now be able to find its own successor and to judge when it is safe
|
||||
to stand aside. Both are between reconciles, which is the only moment the host is not mid-change.
|
||||
- **A machine that is never told a newer version keeps running what it has**, indefinitely and
|
||||
visibly, because its report says which version that is.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A delivered version is run, and the old one is not.** A bed delivers a second version to a machine
|
||||
running the first; the host exits between reconciles, the launcher starts the new one, and the
|
||||
machine reports the new version. This fails against the previous behaviour, where nothing notices a
|
||||
delivered version at all.
|
||||
- **It stands aside between reconciles and never inside one.** Asserted by delivering a version while
|
||||
an apply is in flight: the apply completes, and the exit follows it.
|
||||
- **A version that will not start is rolled back to its predecessor, once**, and the second failure
|
||||
halts with the machine named rather than the binary — asserted with no package manager involved.
|
||||
- **A completed reconcile retires what is older than the predecessor**, and never the predecessor
|
||||
itself, because that is what a rollback needs. Asserted on the directory afterwards.
|
||||
- **The report names the running version**, asserted end to end rather than on the function that reads
|
||||
it, since the point is that the control plane can tell a machine is behind.
|
||||
- **The launcher picks the newest delivered version** with no pointer file and no link, asserted by
|
||||
delivering two and checking which runs.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0005](0005-the-node-host.md) — the host, and what its supervision is for
|
||||
- [ADR 0010](0010-delivery.md) — a declaration is owned resources; this adds no kind to it
|
||||
- [ADR 0012](0012-the-mesh-creates-no-symlinks.md) — why the version is in the path
|
||||
- [ADR 0005](0005-the-node-host.md), *it is built per operating system* — why a rollback written in
|
||||
one package manager's terms was wrong for two of three
|
||||
- [issue 142](../04-ISSUES/142-the-host-is-the-one-thing-the-mesh-does-not-deliver/00-report.md) — the
|
||||
measurement
|
||||
@@ -0,0 +1,158 @@
|
||||
---
|
||||
topic: the mesh
|
||||
status: accepted
|
||||
date: 2026-09-29
|
||||
deciders: jochen
|
||||
reconstructed: false
|
||||
extends: 02-DECISIONS/0141-the-host-delivers-its-own-successor.md
|
||||
---
|
||||
|
||||
# 142. The mesh delivers its own components as binaries, not as container images
|
||||
|
||||
## Context
|
||||
|
||||
Measured on the control-node, 2026-09-29:
|
||||
|
||||
| what | how it runs | publishes |
|
||||
|---|---|---|
|
||||
| host | a binary on the machine | — |
|
||||
| controller, catalogue, builder, vault | containers | nothing |
|
||||
| store, registry, broker | containers | ports |
|
||||
|
||||
**The mesh's own software is delivered two ways, and the difference is not a property of the
|
||||
software.** The host and the controller are both written in the same language, both the mesh's own,
|
||||
both doing the mesh's own work. One is an image fetched from a registry. The other is a file somebody
|
||||
copied to four machines, owned by no package, built by nothing
|
||||
([issue 142](../04-ISSUES/142-the-host-is-the-one-thing-the-mesh-does-not-deliver/00-report.md)).
|
||||
|
||||
**The reason is not a judgement about either, it is that images are the only delivery that works.**
|
||||
There is no way to put a binary on a machine. The host is hand-copied because of that, and the
|
||||
controller is an image because of that. Neither was chosen on its merits.
|
||||
|
||||
What it costs, all of it measured rather than argued:
|
||||
|
||||
- **Genesis must raise a container runtime before the control plane can exist.** The bundle carries
|
||||
three images and one of them is the controller, *"in the bundle for the same reason they are: there
|
||||
is nothing to fetch it with yet"*
|
||||
([design 07](../03-DESIGN/01-to-be/07-the-foundation.md)). So the hardest moment in the mesh's life
|
||||
has a prerequisite that the thing being started does not need.
|
||||
- **Updating the control plane depends on the control plane.** Its image is fetched from the registry,
|
||||
which is a container the controller manages.
|
||||
- **A change to the host cannot be rolled out at all.** Every machine here runs a byte-identical
|
||||
hand-copied binary. A change merged yesterday reached none of them.
|
||||
- **Compiling the language the mesh is written in is not a capability of the builder.** The bundle
|
||||
toolchains are typescript — real, with a registered base module — and python, which is named in the
|
||||
list and absent from the catalogue. The controller is built as an image from a Dockerfile, which is
|
||||
the per-repository incantation the bundle toolchain exists to abolish
|
||||
([design 18](../03-DESIGN/01-to-be/18-building-a-module.md)).
|
||||
|
||||
The half that *receives* a binary safely is already built and tested
|
||||
([ADR 0141](0141-the-host-delivers-its-own-successor.md)): versions side by side in directories named
|
||||
for them, the newest run, the running one standing aside between reconciles, retirement keeping the
|
||||
predecessor, and a rollback that chooses a directory. What is missing is everything that puts one
|
||||
there.
|
||||
|
||||
## Considered Options
|
||||
|
||||
1. **Leave it as it is.** Rejected: it is not a design, it is the reach of one mechanism. And it is
|
||||
what makes a host change undeliverable.
|
||||
2. **Containerise the host too**, so everything is delivered one way. Rejected: the host is what
|
||||
starts the container runtime and what applies containers. A host in a container is the bootstrap
|
||||
problem made total, and the machine would have no way back from a bad one.
|
||||
3. **Deliver the mesh's components as operating-system packages.** Rejected for the reason
|
||||
[ADR 0141](0141-the-host-delivers-its-own-successor.md) rejected it for the host: a package and a
|
||||
trusted repository per operating system, three of each, and the `package` resource asserts presence
|
||||
and deliberately never a version.
|
||||
4. **Binaries for the mesh's own components, containers for third-party software.** Adopted.
|
||||
|
||||
## Decision
|
||||
|
||||
**The mesh's own components are delivered as binaries on the machine.** The host, the controller, the
|
||||
catalogue, the builder, the vault — the software this project writes. They are delivered by the
|
||||
mechanism [ADR 0141](0141-the-host-delivers-its-own-successor.md) built: an archive, fetched by
|
||||
digest, unpacked into a directory named for its version, with the running one standing aside between
|
||||
reconciles and a rollback that chooses the predecessor.
|
||||
|
||||
**Third-party software stays a container.** The store, the registry, the broker. They are somebody
|
||||
else's build, they are already adopted as modules
|
||||
([ADR 0078](0078-the-store-and-broker-are-modules.md)), and an image is the right way to carry
|
||||
somebody else's software. **The container runtime remains required** — modules use it — so this
|
||||
removes a dependency from the control plane, not from the machine.
|
||||
|
||||
**The builder compiles the languages the mesh is written in.** A toolchain for Go, with a base module
|
||||
providing the compiler, exactly as typescript has. The obligation the toolchain list warns about — an
|
||||
SDK carrying the broker client, the envelope and tool serving — attaches to a *module* written in a
|
||||
language, not to the language being compilable. None of these components is a module in that sense;
|
||||
the host is what applies modules.
|
||||
|
||||
**An artifact says what it targets.** A compiled binary is per operating system, pinned at link time
|
||||
([ADR 0005](0005-the-node-host.md)), and a toolchain deliberately takes nothing from the module,
|
||||
because anything a module could override there it would be writing a Dockerfile to override. So the
|
||||
target is a property of the artifact rather than of the recipe, and one artifact declared per target
|
||||
is one build each.
|
||||
|
||||
**A component's version comes from where it sits, not from its linker.** It is unpacked into a
|
||||
directory named for its version, so it can read its own version from its path. The stamp goes, and
|
||||
with it the need for a build to know what it will be called.
|
||||
|
||||
**Genesis carries a binary reference where it carried an image reference.** The principle does not
|
||||
change — the bundle names a thing by digest and the host fetches it, pinned because nothing can
|
||||
resolve a version when no mesh exists — and the container runtime stops being a prerequisite for the
|
||||
control plane. It stays a prerequisite for the store and the broker, which is where it belongs.
|
||||
|
||||
**The order is staged, and each step stands alone.** Compiling Go; an artifact naming its target;
|
||||
delivering a binary; the host as the first component delivered; the controller, catalogue, builder and
|
||||
vault out of their containers; genesis last. Genesis is last for the reason it is always last: it
|
||||
matters for a machine nobody has yet, and every earlier step is provable on a mesh that exists.
|
||||
|
||||
## Consequences
|
||||
|
||||
- **One delivery for the mesh's own software**, so a change to the host ships the way a change to the
|
||||
controller does, and neither is copied by hand.
|
||||
- **The control plane stops depending on a container runtime and on its own registry.** Both remain on
|
||||
the machine for other reasons; neither gates the control plane's own life any more.
|
||||
- **`Replaced()`, the known-good record and the launcher's rollback stop being dead code.** They were
|
||||
written for this and have been called by nothing but their tests.
|
||||
- **Four more components gain a rollback they do not have.** Today a bad controller image is recovered
|
||||
by an operator; under this it is recovered the way a bad host is.
|
||||
- **Two versions of each component occupy disk.** Around nine megabytes each. The predecessor is what a
|
||||
rollback needs.
|
||||
- **Genesis gets smaller, not larger.** One fewer image to carry and one fewer runtime to raise before
|
||||
the control plane.
|
||||
- **This does not make the components smaller or simpler.** They are the same programs; what changes is
|
||||
how they arrive. A reader expecting the containers to have been hiding complexity will not find any.
|
||||
- **What got harder:** the builder gains a language, artifacts gain a target, and the mesh gains a
|
||||
second kind of thing it must deliver correctly — one where getting it wrong takes the control plane
|
||||
down rather than a module. That is why the host is first: it is the component whose recovery is
|
||||
already built and tested.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A component is delivered and runs, with nothing copied by hand.** A bed builds the host from its
|
||||
repository, delivers it to a machine running an older one, and the machine reports the new version.
|
||||
This fails today at the first step, because nothing builds it.
|
||||
- **Each target is built once and only the matching one is delivered.** Asserted by declaring an
|
||||
artifact per operating system and checking that a machine is offered the one it can run — a host
|
||||
built for another is what ADR 0005's link-time pin exists to refuse.
|
||||
- **A component reads its version from its path**, asserted by unpacking the same bytes into two
|
||||
differently named directories and seeing each report its own.
|
||||
- **A bad component is rolled back without an operator**, for the host first: a version that will not
|
||||
start is replaced by its predecessor once, and the second failure halts naming the machine.
|
||||
- **The control plane comes up with no registry reachable**, which is the dependency this removes —
|
||||
asserted by raising it with the registry stopped.
|
||||
- **Genesis raises a control plane with no container runtime running**, and raises the store and the
|
||||
broker afterwards. Last, and on a machine with nothing on it.
|
||||
- **A published port count that does not change.** The mesh's own components publish nothing today, so
|
||||
moving them out of containers must not open anything — asserted on the machine's reachable set before
|
||||
and after, which the converge preview already reads.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0141](0141-the-host-delivers-its-own-successor.md) — the receiving half, already built
|
||||
- [ADR 0005](0005-the-node-host.md) — the host, its supervision, and one binary per operating system
|
||||
- [ADR 0078](0078-the-store-and-broker-are-modules.md) — why third-party software stays a container
|
||||
- [ADR 0006](0006-the-substrate-and-the-control-plane.md) — what genesis must raise, and in what order
|
||||
- [issue 142](../04-ISSUES/142-the-host-is-the-one-thing-the-mesh-does-not-deliver/00-report.md) — the
|
||||
measurement that started this
|
||||
- [design 07](../03-DESIGN/01-to-be/07-the-foundation.md) — the bundle's three images, one of them the
|
||||
controller
|
||||
+22
-1
@@ -134,6 +134,15 @@ python3 00-META/checks/index.py fail if stale
|
||||
- **0106** — [The bus is NATS](0106-the-bus-is-nats.md)
|
||||
- **0116** — [The bus is built in five steps, and the protocol moves with it](0116-the-bus-is-built-in-five-steps.md)
|
||||
- **0119** — [A taken tunnel's predecessor is retired once the take is proven](0119-a-taken-tunnels-predecessor-is-retired.md)
|
||||
- **0125** — [The bus is the only broker](0125-the-bus-is-the-only-broker.md) *(superseded)*
|
||||
- **0127** — [AMQP is a provision, not the bus](0127-amqp-is-a-provision-not-the-bus.md) *(superseded)*
|
||||
- **0128** — [The mesh bus is required, not ambient](0128-the-mesh-bus-is-required-not-ambient.md)
|
||||
- **0129** — [A seat carries the protocol of its role](0129-a-seat-carries-the-protocol-of-its-role.md)
|
||||
- **0130** — [The predecessor is ending, and its broker goes with it](0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md)
|
||||
- **0131** — [Everything on the mesh speaks to the broker seat, and AMQP is not a provision](0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)
|
||||
- **0132** — [A seat carries the tools its holder must serve](0132-a-seat-carries-the-tools-its-holder-must-serve.md)
|
||||
- **0134** — [The mesh says what it applied](0134-the-mesh-says-what-it-applied.md)
|
||||
- **0142** — [The mesh delivers its own components as binaries, not as container images](0142-the-mesh-delivers-its-own-components-as-binaries.md)
|
||||
|
||||
### Its tiers, from the bottom up
|
||||
|
||||
@@ -164,6 +173,7 @@ python3 00-META/checks/index.py fail if stale
|
||||
- **0098** — [A fact a provider makes at first start is fetched from it, not carried in its manifest](0098-a-fact-a-provider-makes-at-first-start-is-fetched-from-it.md)
|
||||
- **0108** — [A route carries the policy applied to a request, and names a secret rather than holding one](0108-a-route-carries-the-policy-applied-to-a-request.md)
|
||||
- **0109** — [A package registry seat is one per ecosystem, not one for all of them](0109-a-package-registry-seat-is-one-per-ecosystem.md)
|
||||
- **0126** — [A module declares its own seats; the mesh reserves its own](0126-a-module-declares-its-own-seats.md)
|
||||
|
||||
### What runs on them, and how it gets there
|
||||
|
||||
@@ -196,12 +206,23 @@ python3 00-META/checks/index.py fail if stale
|
||||
- **0091** — [A mount is declared, and there are three things it can be](0091-a-mount-is-declared-three-ways.md)
|
||||
- **0099** — [A step that runs once names what it reads, and runs again when it changed](0099-a-step-that-runs-once-names-what-it-reads.md)
|
||||
- **0110** — [A seat is held by one assignment, from a closed set, and it may deliver a provision](0110-a-seat-is-a-module-assignment-from-a-closed-set.md)
|
||||
- **0112** — [A module definition names no node, no mesh and no path: everything it needs is a requirement the mesh resolves](0112-a-module-definition-names-no-node-mesh-or-path.md) *(proposed)*
|
||||
- **0112** — [A module definition names no node, no mesh and no path: everything it needs is a requirement the mesh resolves](0112-a-module-definition-names-no-node-mesh-or-path.md)
|
||||
- **0113** — [The vault makes every shared secret, a provider makes resources and data, and the mesh carries both](0113-the-vault-makes-every-secret.md) *(proposed)*
|
||||
- **0114** — [A credential two parties hold rotates over two credentials; one a single party holds rotates in place, staged; and retiring a credential never removes what it reached](0114-a-shared-credential-rotates-over-two-credentials.md) *(proposed)*
|
||||
- **0115** — [One assignment of a module per node: the module's name is the assignment's identity](0115-one-assignment-of-a-module-per-node.md) *(proposed)*
|
||||
- **0117** — [A machine's uplink is a seat: the mesh configures the manager, never the link](0117-a-machines-uplink-is-a-seat.md)
|
||||
- **0118** — [Undeclaring removes what the mesh made, and gives a unit back the state it was found in](0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md)
|
||||
- **0120** — [A roster fact carries its format as a template: the mesh owns the data, the module owns the format](0120-a-roster-fact-carries-its-format-as-a-template.md)
|
||||
- **0121** — [A system seat is named for its scope, and a module may define its own](0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md)
|
||||
- **0122** — [A seat is data the controller owns, and a rename is a database update](0122-a-seat-is-data-a-rename-is-a-database-update.md)
|
||||
- **0133** — [A module owns its migrations, and the mesh owns when they run](0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md) *(superseded)*
|
||||
- **0135** — [A module version prepares its state before it runs](0135-a-module-version-prepares-its-state-before-it-runs.md)
|
||||
- **0136** — [A step gates its module, not the machine](0136-a-step-gates-its-module-not-the-machine.md)
|
||||
- **0137** — [A machine says which networks it routes](0137-a-machine-says-which-networks-it-routes.md) *(superseded)*
|
||||
- **0138** — [An assignment binds an endpoint and says how far it reaches](0138-an-assignment-binds-an-endpoint-and-says-how-far-it-reaches.md)
|
||||
- **0139** — [A network is forwarded because a module declared it](0139-a-network-is-forwarded-because-a-module-declared-it.md) *(superseded)*
|
||||
- **0140** — [The filter constrains what arrives from outside, and says nothing about a machine's own guests](0140-the-filter-constrains-what-arrives-from-outside.md)
|
||||
- **0141** — [The host delivers its own successor, and versions live side by side](0141-the-host-delivers-its-own-successor.md)
|
||||
|
||||
### How it is built
|
||||
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
layer: to-be
|
||||
status: in-progress
|
||||
code: [mesh-host]
|
||||
updated: 2026-09-22
|
||||
updated: 2026-09-29
|
||||
decisions:
|
||||
- 02-DECISIONS/0141-the-host-delivers-its-own-successor.md
|
||||
- 02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md
|
||||
- 02-DECISIONS/0102-the-mesh-writes-into-a-shared-file-never-over-it.md
|
||||
- 02-DECISIONS/0103-what-an-adopted-node-holds-and-what-its-guard-refuses.md
|
||||
@@ -423,3 +424,45 @@ ignored an instruction and "applied" would be a lie. Applying stays one at a tim
|
||||
is not. **Checked** by the link's unit tests on the drain, and by the genesis bed's settle wait,
|
||||
which counts on a node catching up to the newest declaration rather than the oldest.
|
||||
|
||||
## The host delivers its own successor
|
||||
|
||||
*2026-09-29, from a change to the host that could reach no machine —
|
||||
[issue 142](../../04-ISSUES/142-the-host-is-the-one-thing-the-mesh-does-not-deliver/00-report.md),
|
||||
settled by [ADR 0141](../../02-DECISIONS/0141-the-host-delivers-its-own-successor.md).*
|
||||
|
||||
A merge builds every changed module and the control plane, and the result reaches the machines running
|
||||
it with nobody asking. The host was the exception: not a build target, named by no declaration, and
|
||||
identical on every machine because somebody had copied it there.
|
||||
|
||||
The supervision needed for this was already right. A clean exit from the host means it has stood aside,
|
||||
and the launcher's next turn runs whatever is on disk. Consecutive failed starts are counted, a
|
||||
rollback happens at the limit, and a second failure halts with the machine named rather than the binary.
|
||||
What was missing was smaller than it looked: nothing told the running host a successor was waiting, and
|
||||
the rollback resolved its known-good *version* through one operating system's package manager, which no
|
||||
machine here used.
|
||||
|
||||
Keeping a version rather than a path was the clue. That is only useful to something that can choose
|
||||
between versions present on the machine — so the versions live side by side:
|
||||
|
||||
- **A version arrives as an archive, in a directory named for it.** The ordinary resource, fetched by
|
||||
digest and checked before anything is unpacked. The path written is never the path being executed, so
|
||||
replacing a running binary — which the kernel refuses — never comes up.
|
||||
- **The launcher starts the most recently delivered version**, reading no pointer and following no
|
||||
link, because the version is in the path.
|
||||
- **The running host stands aside between reconciles and never inside one.** Standing aside mid-apply is
|
||||
the half-configured machine this document exists to prevent.
|
||||
- **A version that completes a reconcile records itself and retires what is older than its
|
||||
predecessor.** The predecessor stays, because that is what a rollback needs.
|
||||
- **Rollback starts that predecessor** instead of reinstalling a package: no package manager, no cache
|
||||
somebody else may clean, and the same script on every operating system.
|
||||
- **A machine says which host version it runs**, on the report it already sends, so being behind is
|
||||
answerable at all.
|
||||
|
||||
One copy by hand remains, once: the first host that understands versioned directories cannot be fetched
|
||||
by a host that does not.
|
||||
|
||||
*How it is checked* is stated with the decision — a second version delivered to a running machine is
|
||||
run and reported; the exit follows an in-flight apply rather than interrupting it; a version that will
|
||||
not start is rolled back once and the second failure halts; a completed reconcile retires what is older
|
||||
than the predecessor and never the predecessor; and the newest of two delivered versions is the one
|
||||
that runs.
|
||||
|
||||
@@ -11,8 +11,9 @@ code:
|
||||
- mesh-catalog modules/postgres
|
||||
- mesh-catalog modules/lavinmq
|
||||
- mesh-lab test/integration/mesh.test.ts (a bare machine becomes a mesh)
|
||||
updated: 2026-09-22
|
||||
updated: 2026-09-29
|
||||
decisions:
|
||||
- 02-DECISIONS/0142-the-mesh-delivers-its-own-components-as-binaries.md
|
||||
- 02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md
|
||||
- 02-DECISIONS/0088-the-foundation-filters-before-anything-listens.md
|
||||
- 02-DECISIONS/0004-a-node-and-how-it-joins.md
|
||||
@@ -314,4 +315,24 @@ of a database and pushed to over the broker. What arrived and what did not is th
|
||||
|
||||
**One fault, and it was in the joining.** The token did not say what the mesh calls the machine,
|
||||
so enrolment needed a flag its own help said it did not — and failed at the broker with an empty
|
||||
username. Recorded in ADR 0004 as the fifth thing a token carries.
|
||||
username. Recorded in ADR 0004 as the fifth thing a token carries.
|
||||
|
||||
## The mesh's own components arrive as binaries
|
||||
|
||||
*2026-09-29 —
|
||||
[ADR 0142](../../02-DECISIONS/0142-the-mesh-delivers-its-own-components-as-binaries.md).*
|
||||
|
||||
The bundle carries three images and one of them is the controller, *because there is nothing to fetch
|
||||
it with yet*. That reasoning holds and its conclusion changes: the controller is carried as a **binary**
|
||||
reference rather than an image reference, pinned by digest exactly as before. Nothing about the bundle's
|
||||
shape moves — it names a thing and the host fetches it — and the container runtime stops being something
|
||||
genesis must raise before the control plane can exist. It still raises one, for the store and the broker,
|
||||
which is where somebody else's software belongs.
|
||||
|
||||
The mesh's own components — the host, the controller, the catalogue, the builder, the vault — are
|
||||
delivered as binaries into directories named for their versions, by the mechanism
|
||||
[ADR 0141](../../02-DECISIONS/0141-the-host-delivers-its-own-successor.md) describes. Third-party
|
||||
software stays a container. The split is not about isolation; it is about who built the thing.
|
||||
|
||||
Measured before deciding it: the mesh's own components publish no ports at all, so this opens nothing.
|
||||
Only the store, the registry and the broker publish, and they are staying as they are.
|
||||
|
||||
@@ -7,8 +7,10 @@ code:
|
||||
- mesh-controller internal/identity/authority.go
|
||||
- mesh-host internal/identity/serving.go
|
||||
- mesh-host internal/apply (the service that reflects a rule set)
|
||||
updated: 2026-09-27
|
||||
updated: 2026-09-28
|
||||
decisions:
|
||||
- 02-DECISIONS/0138-an-assignment-binds-an-endpoint-and-says-how-far-it-reaches.md
|
||||
- 02-DECISIONS/0140-the-filter-constrains-what-arrives-from-outside.md
|
||||
- 02-DECISIONS/0104-a-provision-may-be-answered-by-an-adapter-to-the-predecessor.md
|
||||
- 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
- 02-DECISIONS/0105-the-mesh-adopts-the-predecessors-tunnel-in-place.md
|
||||
@@ -625,6 +627,50 @@ the found firewall reloads and reachable from a container on the node, that a ma
|
||||
enrols through the openings before and after a reload and a reboot, and that after the flip the
|
||||
declared port is open and the undeclared one closed.
|
||||
|
||||
### It filters what arrives from outside, and not what the machine's own guests send
|
||||
|
||||
*2026-09-28, preparing the control-node's convergence —
|
||||
[issue 141](../../04-ISSUES/141-the-forward-chain-does-not-follow-the-modules/00-report.md), settled by
|
||||
[ADR 0140](../../02-DECISIONS/0140-the-filter-constrains-what-arrives-from-outside.md), which replaces
|
||||
[ADR 0137](../../02-DECISIONS/0137-a-machine-says-which-networks-it-routes.md) and
|
||||
[ADR 0139](../../02-DECISIONS/0139-a-network-is-forwarded-because-a-module-declared-it.md).*
|
||||
|
||||
Traffic passing through a machine is filtered, because a container's published port is traffic passing
|
||||
through rather than traffic arriving at the machine itself. Having blocked it, the filter then had to
|
||||
let the machine's own containers reach outward again — and it did that by listing the address ranges
|
||||
they sit on. Two of those ranges were constants in this repository's code, and the rest were typed by an
|
||||
operator after the flip had already cut a workstation's containers off from everything.
|
||||
|
||||
**The list was the mistake, not its contents.** A constant describes one machine. A typed range goes
|
||||
stale and cannot tell a network the mesh made from one a predecessor left behind — on the control-node,
|
||||
six ranges fall outside the constants and two of the six belong to services the mesh does not run. The
|
||||
attempt to generate the list from the modules put half the rule set on the machine and made the
|
||||
derivation advisory. Three records, one list.
|
||||
|
||||
**And the mesh has no position on a container reaching outward.** §4 exists to say which port is open
|
||||
and to whom, which is about what arrives. A container of this machine's own opening a connection
|
||||
somewhere is not a port opened to anybody, and the addresses it might do that from are the machine's
|
||||
internal plumbing, which the mesh neither owns nor can know.
|
||||
|
||||
So the filter constrains what arrives from **outside** the machine and says nothing about what did not.
|
||||
Traffic passing through is allowed unless it came in on one of the machine's outward links, and then
|
||||
only where a declared endpoint's reach admits it (§6). The machine says which of its links face
|
||||
outside — one fact it reports, like the kind of firewall it found and the tunnel it carries, not a
|
||||
setting and not a list of addresses. It does not change when a module is added or removed, which is the
|
||||
whole difference from what it replaces. A machine that has reported no outward link is sent no filter
|
||||
at all, and keeps the one it has, because a rule written around a link with no name is a rule set that
|
||||
does not load — a machine filtering nothing while its unit reports success.
|
||||
|
||||
Ports go on following the modules exactly as before: assign a module to a machine and the port its
|
||||
assignment says it reaches on opens. Nothing about a network is said anywhere, by anybody.
|
||||
|
||||
*How it is checked:* a bed converges a machine carrying containers on several networks, none of them
|
||||
named in any setting, and each reaches outward afterwards — which fails against the previous behaviour,
|
||||
where the same flip cut them off, and is how it was written; a network made *after* the last declaration
|
||||
needs no new filter; a declared port is reachable from off the private network and an undeclared one is
|
||||
not; no address of a machine's own networks appears in a rendered filter, asserted on the text; and a
|
||||
machine reporting no outward link is refused in the control plane with its existing filter left alone.
|
||||
|
||||
## 5 — Certificates
|
||||
|
||||
**Two authorities, kept separate on purpose.**
|
||||
@@ -710,6 +756,60 @@ that verifies against the internal root and nothing else — which cannot succee
|
||||
first reached the name to certify it*
|
||||
([ADR 0066](../../02-DECISIONS/0066-public-routing-is-name-agnostic.md)).
|
||||
|
||||
## 6 — One statement behind exposure, filtering and certificates
|
||||
|
||||
*2026-09-28, preparing the control-node's convergence —
|
||||
[issue 140](../../04-ISSUES/140-an-endpoints-reach-is-not-declared/00-report.md), settled by
|
||||
[ADR 0138](../../02-DECISIONS/0138-an-assignment-binds-an-endpoint-and-says-how-far-it-reaches.md).*
|
||||
|
||||
The three sections above each decide, independently, how far a service reaches. §3 composes a name
|
||||
from a label and the node's domain. §4 opens a port to the source a listen named. §5 certifies the
|
||||
names that exist, from whichever authority the proxy holds. Each is coherent on its own, and together
|
||||
they mean **reachability is never stated anywhere** — it is the sum of three derivations, and a sum is
|
||||
not something anyone can review or refuse.
|
||||
|
||||
What that costs, measured: an identity provider holding a public certificate valid 90 days and an
|
||||
internal one valid 24 hours, neither asked for by any assignment, because both names existed and a
|
||||
proxy certifies what it serves. And an endpoint that is not routed — git over ssh — which can be
|
||||
spoken about only in the filter's vocabulary, so *this must be reachable from outside* is a setting
|
||||
exactly one mechanism reads.
|
||||
|
||||
**An endpoint is the thing that was missing.** A module declares named endpoints: one port it serves,
|
||||
what it is for, and what it would serve that to absent any instruction. A route contribution names an
|
||||
endpoint rather than repeating a port number. An assignment — which is where a module's configuration
|
||||
lives ([ADR 0046](../../02-DECISIONS/0046-a-module-configuration-is-its-assignments-not-its-manifest.md))
|
||||
— then binds each endpoint to a machine port and says how far it reaches.
|
||||
|
||||
One value, three readers:
|
||||
|
||||
| reach | the filter opens | the proxy serves | the certificate comes from |
|
||||
|---|---|---|---|
|
||||
| `internal` | the machine port, to the private network | the internal name | the mesh's own authority |
|
||||
| `public` | the machine port, to anywhere | the public name | the public authority |
|
||||
| `both` | the machine port, to anywhere | both names | each name's own authority |
|
||||
|
||||
**An endpoint that is not routed is reached and never named.** No route contribution means no name is
|
||||
composed and no certificate requested, while the filter still acts on it. That is the case the model
|
||||
could not express at all, and it is the ordinary case for anything that is not HTTP.
|
||||
|
||||
**Nothing moves until an assignment says so.** An endpoint whose assignment is silent keeps the
|
||||
default its manifest states, so every machine already converged renders exactly as it does today —
|
||||
the same property §4 needed when a machine gained a way to say which networks it routes.
|
||||
|
||||
This is what the certificate questions were waiting for. Which authority signs a name, whether a name
|
||||
may appear in a public issuance log, and what must be trusted where are all answerable once an
|
||||
endpoint says whether it is internal — and unanswerable while the proxy decides by composing every
|
||||
name it can. It is also the fact
|
||||
[issue 139](../../04-ISSUES/139-an-internal-route-name-resolves-to-the-consumers-node/00-report.md)
|
||||
needs: an internal name should be composed from the machine serving the endpoint, which is the
|
||||
assignment that bound it.
|
||||
|
||||
**How it is checked** is stated with the decision: one module with two endpoints of differing reach
|
||||
asserted per chain body; the names and the certificate requests following the reach and failing
|
||||
against today's behaviour, where both are always composed; an unrouted endpoint filtered and never
|
||||
named; an assignment naming an endpoint the module does not declare refused where it is said; and a
|
||||
silent assignment rendering byte-identically to today.
|
||||
|
||||
## What this removes
|
||||
|
||||
The list is worth having in one place, because it is most of the argument:
|
||||
@@ -769,7 +869,7 @@ where a found tunnel is left running beside the mesh's; where it is adopted ther
|
||||
The guard admits the mesh's ports from that one interface, and the predecessor's peers arrive on
|
||||
it.
|
||||
|
||||
*2026-09-27, [ADR 0119](../../02-DECISIONS/0119-a-taken-tunnels-predecessor-is-retired.md).* The
|
||||
*2026-09-27, [ADR 0127](../../02-DECISIONS/0119-a-taken-tunnels-predecessor-is-retired.md).* The
|
||||
found configuration is kept only until the take is proven — the found unit down, the mesh's
|
||||
interface up and handshaking with a peer. Then it is removed from where the found unit reads it
|
||||
(its original stays kept), the hold ends, and the predecessor's tunnel cannot be raised again by
|
||||
|
||||
@@ -5,8 +5,9 @@ code:
|
||||
- mesh-controller cmd/mesh-builder
|
||||
- mesh-controller internal/builder
|
||||
- mesh-catalog modules/builder
|
||||
updated: 2026-09-25
|
||||
updated: 2026-09-29
|
||||
decisions:
|
||||
- 02-DECISIONS/0142-the-mesh-delivers-its-own-components-as-binaries.md
|
||||
- 02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md
|
||||
- 02-DECISIONS/0097-a-vendor-image-is-a-declared-build-input.md
|
||||
- 02-DECISIONS/0096-an-upstream-image-is-copied-between-registries.md
|
||||
@@ -263,3 +264,27 @@ ships one and wrong for code the mesh built, which has no unit until the mesh wr
|
||||
|
||||
**Tools, hooks and consumers are not further modes**, which is the test of whether three is the
|
||||
right number: they are loaded by a tool host, and a tool host is a process that stays up.
|
||||
|
||||
## The builder compiles the languages the mesh is written in
|
||||
|
||||
*2026-09-29 —
|
||||
[ADR 0142](../../02-DECISIONS/0142-the-mesh-delivers-its-own-components-as-binaries.md).*
|
||||
|
||||
The toolchain list was typescript and python, and only typescript had a base module in the catalogue.
|
||||
Meanwhile the control plane — written in the language this project is mostly written in — was built as
|
||||
an image from a hand-written Dockerfile, which is the per-repository incantation this whole mechanism
|
||||
exists to abolish.
|
||||
|
||||
So the list gains Go, with a base module providing the compiler exactly as typescript has one. The
|
||||
obligation the list's own comment warns about — an SDK carrying the broker client, the event envelope
|
||||
and tool serving — attaches to a **module** written in a language, not to the language being
|
||||
compilable. The mesh's own components are not modules in that sense; the host is what applies modules.
|
||||
|
||||
**And an artifact says what it targets.** A compiled binary is per operating system, pinned at link
|
||||
time, and a toolchain deliberately accepts nothing from the module — anything a module could override
|
||||
there it would be writing a Dockerfile to override. The target is therefore a property of the artifact,
|
||||
not of the recipe: one artifact declared per target, one build each.
|
||||
|
||||
A component's version stops being stamped in at link time. It is unpacked into a directory named for
|
||||
its version, so it reads its version from its own path, and a build no longer has to know what it will
|
||||
be called.
|
||||
|
||||
@@ -3,12 +3,13 @@ layer: to-be
|
||||
status: proposed
|
||||
code:
|
||||
- mesh-sdk src
|
||||
- mesh-tools src/broker-amqp.ts
|
||||
- mesh-tools src/broker-nats.ts (and broker-amqp.ts until the rollout)
|
||||
- mesh-controller internal/link
|
||||
updated: 2026-09-26
|
||||
decisions:
|
||||
- 02-DECISIONS/0095-the-control-plane-is-the-way-to-ask-a-module.md
|
||||
- 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
- 02-DECISIONS/0128-the-mesh-bus-is-required-not-ambient.md
|
||||
- 02-DECISIONS/0116-the-bus-is-built-in-five-steps.md
|
||||
- 02-DECISIONS/0074-the-wire-is-specified-not-the-types.md
|
||||
- 02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md
|
||||
@@ -25,26 +26,17 @@ language and nothing more ([ADR 0074](../../02-DECISIONS/0074-the-wire-is-specif
|
||||
This is a specification, so it says what is required rather than how anything is arranged. Where it
|
||||
describes current behaviour that is *not yet* specified-and-conformed, it says so.
|
||||
|
||||
> **The wire below is the bus being replaced.** *2026-09-26.*
|
||||
> [ADR 0106](../../02-DECISIONS/0106-the-bus-is-nats.md) moved the mesh's bus to NATS. Everything
|
||||
> in this document that names an exchange, a queue or a routing key — the event exchanges, the
|
||||
> durable `<node>.<module>.events` queue, the shared `serve.<key>` queue — describes the transport
|
||||
> being retired, and the conformance fixtures were captured against it.
|
||||
> **Rewritten onto NATS, 2026-09-26** (step 3 of
|
||||
> [ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md)). What
|
||||
> [ADR 0074](../../02-DECISIONS/0074-the-wire-is-specified-not-the-types.md) decided is untouched:
|
||||
> a floor plus independent capabilities, an implementation legitimate when it claims less,
|
||||
> identity from the sealed credential, at-least-once with dedup on `x-event-id`, and conformance
|
||||
> as executable fixtures rather than prose. What changed is the transport beneath all of it —
|
||||
> exchanges and queues became subjects and streams. The envelope keeps its shape
|
||||
> ([ADR 0042](../../02-DECISIONS/0042-the-shape-of-an-event-on-the-wire.md)).
|
||||
>
|
||||
> What does **not** change is this document's model, which is the part ADR 0074 decided: a floor
|
||||
> plus independent capabilities, an SDK that implements what it claims and is legitimate when it
|
||||
> claims less, identity taken from the sealed credential rather than the environment, at-least-once
|
||||
> with dedup on `x-event-id`, and conformance as executable fixtures per capability rather than
|
||||
> prose. The envelope keeps its shape ([ADR 0042](../../02-DECISIONS/0042-the-shape-of-an-event-on-the-wire.md));
|
||||
> it becomes the message body.
|
||||
>
|
||||
> Rewriting the wire sections onto the subjects and streams of
|
||||
> [design 25](25-the-bus-on-nats.md) §2–§3, and recapturing the fixtures there, is **step 3 of
|
||||
> [ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md)**. Until that lands, read
|
||||
> the sections below for what two implementations may not disagree *about*, and design 25 for what
|
||||
> they will disagree about it *on*. A specification that silently described a retired transport
|
||||
> would be worse than an absent one, because it reads as current — hence this note rather than a
|
||||
> quiet edit.
|
||||
> Statements here marked *verified* were checked against a running server while the runtime's
|
||||
> client was written, not reasoned from documentation.
|
||||
|
||||
## The shape of it
|
||||
|
||||
@@ -70,22 +62,39 @@ document:
|
||||
|
||||
| field | is | required |
|
||||
|---|---|---|
|
||||
| `url` | an `amqps://` URL carrying the account's user and password | yes |
|
||||
| `fingerprint` | sha256 of the certificate the broker must present | yes for a scoped account |
|
||||
| `url` | a `tls://` URL for the bus, with the account's user and password | yes |
|
||||
| `fingerprint` | sha256 of the certificate the bus must present | yes for a scoped account |
|
||||
| `node` | the machine this account was issued for | yes for a scoped account |
|
||||
| `module` | the module this account was issued for | yes for a scoped account |
|
||||
|
||||
A plain string rather than a document is a **bootstrap URL** — unscoped, for the moment before a
|
||||
mesh can issue anything. An implementation accepts both and must not treat the second as ordinary.
|
||||
|
||||
**`node` and `module` are not decoration: every subject an implementation touches is derived from
|
||||
them.** Its own namespace is `mesh.mod.<module>`, its consumer is `<node>_<module>`, its inbox is
|
||||
its own. So a credential without them is refused rather than guessed at — an implementation that
|
||||
fell back to an environment variable would let anything on the machine decide which module it is,
|
||||
which is what the identity rule below exists to prevent.
|
||||
|
||||
The credential itself is fetched, never carried in a declaration: a declaration is persisted as
|
||||
state and a sealed secret in a stream is an archive rather than a moment
|
||||
([design 29](32-what-a-module-declares.md) §10).
|
||||
|
||||
### Connecting
|
||||
|
||||
- The connection **pins the fingerprint**. It does not trust a certificate authority, and it does
|
||||
not skip verification. A broker presenting a different certificate is refused, whatever else is
|
||||
not skip verification. A bus presenting a different certificate is refused, whatever else is
|
||||
true of it.
|
||||
- A scoped account **does not declare exchanges**. The foundation owns them; an account that may
|
||||
declare one is an account that may create a parallel mesh by typo.
|
||||
- An implementation **declares its own queue** and nothing else.
|
||||
- **The certificate must also carry a name the bus is dialled by.** *Verified:* the NATS client
|
||||
exposes no hook to replace hostname verification, so pinning no longer makes it redundant the
|
||||
way it did on AMQP — the pin happens before dialling and the library's own name check happens
|
||||
beside it. A certificate without a matching subject-alternative name is refused at connect, by
|
||||
a library error rather than by anything the mesh says.
|
||||
- An implementation **creates nothing on the bus**: not a stream, not a consumer, not a subject.
|
||||
Streams and durable consumers are the controller's alone ([design 25](25-the-bus-on-nats.md)
|
||||
§3), and a module's account cannot reach the JetStream API to make one. An implementation binds
|
||||
the consumer the mesh created for it, and if it is absent that is a mesh that has not finished
|
||||
assigning the module, not something for the module to fix.
|
||||
|
||||
### Identity
|
||||
|
||||
@@ -100,26 +109,49 @@ the credential disagree, the credential wins and the variable is overwritten.
|
||||
|
||||
## Capability: events
|
||||
|
||||
### The exchanges
|
||||
### The subjects
|
||||
|
||||
| exchange | carries |
|
||||
| subject | carries |
|
||||
|---|---|
|
||||
| `mesh.events` | every event |
|
||||
| `mesh.events.dead` | what could not be handled |
|
||||
| `mesh.mod.<module>.event.<key>` | an event that module emitted |
|
||||
| `mesh.seat.<seat>.event.<verb>` | an event the holder of that role emitted |
|
||||
|
||||
### The queue
|
||||
Both are captured by the `EVENTS` stream. **An event's source is enforced rather than claimed**: a
|
||||
module's account may publish only into its own namespace, so `x-source` cannot disagree with where
|
||||
the message arrived from.
|
||||
|
||||
One **durable** queue per consumer, named `<node>.<module>.events`, with as many bindings as the
|
||||
module has patterns. Durable because an event emitted while a module is restarting is exactly the
|
||||
one that must not be lost.
|
||||
**The `event` token is load-bearing.** A module's namespace also carries its tool calls
|
||||
(`mesh.mod.<module>.tool.<tool>`), and a stream is defined by a subject filter — without the token
|
||||
the events stream would capture every tool invocation in the mesh, and a tool call must never be
|
||||
persisted.
|
||||
|
||||
**A message matching two bindings is delivered once**, so an implementation must match the routing
|
||||
key against its own patterns locally to decide which handlers run. An implementation that ran every
|
||||
handler whose exchange binding matched would run the wrong one.
|
||||
### The consumer
|
||||
|
||||
One **durable consumer** per module, named `<node>_<module>`, carrying one filter per pattern the
|
||||
module consumes. Durable because an event emitted while a module is restarting is exactly the one
|
||||
that must not be lost.
|
||||
|
||||
**Created by the controller, bound by the implementation.** A module declares what it reacts to
|
||||
and never how delivery works, so it does not name its consumer, does not choose its ack policy or
|
||||
delivery limit, and cannot misconfigure them.
|
||||
|
||||
*Verified, and it is a trap:* a durable name **may not contain a dot**, while the subject a
|
||||
consumer acknowledges on is `$JS.ACK.<stream>.<consumer>.…` — two names joined by one. An
|
||||
implementation that treats them as a single string reads correctly in a permission list and is
|
||||
refused as a consumer name. Left wrong, the symptom is every message redelivered forever while
|
||||
the permissions look right.
|
||||
|
||||
**One consumer may carry filters wider than one handler's pattern**, because a module subscribing
|
||||
twice gets one consumer with both. So an implementation still matches the key against its own
|
||||
patterns locally to decide which handlers run — and **acknowledges a message no handler wanted**,
|
||||
or it is redelivered until it expires.
|
||||
|
||||
### The envelope
|
||||
|
||||
Headers ride as AMQP headers. The body is JSON.
|
||||
Headers ride as **NATS headers**; the body is JSON, and the body alone. *Verified:* the payload is
|
||||
the event's `body`, not the whole envelope re-encoded — an implementation that nested the envelope
|
||||
would pass every one of its own tests and agree with no other, which is the exact failure the
|
||||
conformance fixtures exist to catch. The key is recovered from the subject, not carried twice.
|
||||
|
||||
| header | is | required |
|
||||
|---|---|---|
|
||||
@@ -140,6 +172,11 @@ breaking change for everybody.
|
||||
At-least-once. **Deduplication is on `x-event-id`**, which only the emitter can produce — a
|
||||
consumer cannot tell a redelivery from a second event any other way.
|
||||
|
||||
On NATS the id does double duty: an implementation passes it as the publish's message id, so the
|
||||
**server** also refuses a duplicate inside its window. That narrows the window in which a
|
||||
consumer has to deduplicate; it does not remove the requirement, because the window is finite and
|
||||
a redelivery after it is still a redelivery.
|
||||
|
||||
### What is true, checked (2026-09-16)
|
||||
|
||||
Go emits all five required headers; the SDK requires exactly those. `x-causation-id` and `x-schema`
|
||||
@@ -154,22 +191,30 @@ version to declare.
|
||||
|
||||
A module's tools are its operator-facing surface.
|
||||
|
||||
- A tool is served from a **shared durable queue**, `serve.<key>`. Shared, so several runtimes
|
||||
serving one tool compete for a call rather than each answering it.
|
||||
- A call is request and reply. The reply returns through the RPC exchange `mesh.rpc`, keyed by the
|
||||
caller's own reply queue — **not** through the default exchange, which would let a caller publish
|
||||
into any queue on the broker.
|
||||
- A caller needs a **reply queue**, and that is what a module's scoped account may not declare
|
||||
([issue 049](../../04-ISSUES/049-a-module-can-serve-tools-and-nothing-can-call-them/00-report.md)).
|
||||
So a module may serve tools and may not call them.
|
||||
- **The control plane is the way to ask**
|
||||
([ADR 0095](../../02-DECISIONS/0095-the-control-plane-is-the-way-to-ask-a-module.md)):
|
||||
`ask <module> <tool> [json]` publishes on `mesh.rpc` under `<module>.<tool>` with a private reply
|
||||
queue bound under its own name, and prints the answer as the module gave it. A module declares
|
||||
nothing about being asked — serving a tool is being askable through the control plane. A
|
||||
module-to-module call, if one is wanted, is a grant like any other and a later decision.
|
||||
*How it is checked:* a tools-only bed asks a served tool through the control plane and asserts
|
||||
an answer arrived, where a timeout would read differently.
|
||||
- A tool is served on `mesh.mod.<module>.tool.<tool>`, with a **queue group** — so several
|
||||
runtimes serving one tool compete for a call rather than each answering it.
|
||||
- A call is request and reply on **core NATS, never a stream**. A tool call is not persisted: a
|
||||
lost one is a timeout the caller already handles, and a stream of them would be the mesh's most
|
||||
voluminous and least valuable traffic competing for retention with the messages that matter.
|
||||
- The reply goes to the inbox the request carries. A responder may answer it because its account
|
||||
is granted **`allow_responses`** — one reply to the subject of a message it actually received,
|
||||
and nothing wider. That is what makes a per-account inbox prefix workable: no user is ever
|
||||
granted `_INBOX.>`, so without it a responder could not reach the caller at all.
|
||||
- **A module may now call a tool, which on AMQP it could not.** *Verified:* two modules on
|
||||
separate connections, one serving and one calling, with an answer returned and a throwing
|
||||
handler reaching the caller as an error rather than a timeout.
|
||||
[Issue 049](../../04-ISSUES/049-a-module-can-serve-tools-and-nothing-can-call-them/00-report.md)
|
||||
recorded the old limit — a scoped account could not declare the reply queue a caller needs —
|
||||
and [ADR 0095](../../02-DECISIONS/0095-the-control-plane-is-the-way-to-ask-a-module.md) routed
|
||||
every ask through the control plane because of it. **That constraint is gone**, and each
|
||||
account's own inbox prefix replaces it.
|
||||
|
||||
ADR 0095 is not thereby reversed: the control plane remains *a* way to ask, and a person asking
|
||||
a module should still go through it. What changes is that "a module-to-module call, if one is
|
||||
wanted, is a later decision" is no longer a question about *capability*. It is a policy
|
||||
question, and the answer the mesh already has is `uses`: a module declares the seat it calls,
|
||||
and the permission follows the declaration.
|
||||
- A module declares nothing about being asked — serving a tool is being askable.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ updated: 2026-09-25
|
||||
decisions:
|
||||
- 02-DECISIONS/0084-which-provider-serves-a-consumer.md
|
||||
- 02-DECISIONS/0027-a-provision-names-what-the-consumer-is-coupled-to.md
|
||||
- 02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
- 02-DECISIONS/0126-a-module-declares-its-own-seats.md
|
||||
---
|
||||
|
||||
# 23 — Choosing a provider
|
||||
@@ -54,7 +54,7 @@ to that provider and not to whichever one is nearest.
|
||||
**A seat names the mesh's one provider of a kind.** Where a seat delivers the provision, its holder
|
||||
answers for it when several providers exist and the consumer named none. That is not picking: the
|
||||
choice was made once, mesh-wide, by assigning the holder, rather than once per consumer by naming it
|
||||
([ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md),
|
||||
([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)),
|
||||
[26 — The seats](26-the-seats.md)). A named provider still wins over the seat, because a consumer
|
||||
coupled to particular contents has said so.
|
||||
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: proposed
|
||||
status: in-progress
|
||||
code:
|
||||
- mesh-controller internal/link (to be replaced)
|
||||
- mesh-host internal/link (to be replaced)
|
||||
- mesh-tools src/broker-amqp.ts (to be replaced)
|
||||
- mesh-catalog modules/nats (to be written)
|
||||
- mesh-sdk src (the protocol's NATS binding, step 3)
|
||||
updated: 2026-09-26
|
||||
updated: 2026-09-27
|
||||
decisions:
|
||||
- 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
- 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
- 02-DECISIONS/0116-the-bus-is-built-in-five-steps.md
|
||||
- 02-DECISIONS/0074-the-wire-is-specified-not-the-types.md
|
||||
- 02-DECISIONS/0079-the-foundation-seats-are-named-after-their-servers.md
|
||||
@@ -17,6 +18,8 @@ decisions:
|
||||
- 02-DECISIONS/0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md
|
||||
- 02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md
|
||||
- 02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md
|
||||
- 02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md
|
||||
- 02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md
|
||||
---
|
||||
|
||||
# 25. The bus on NATS
|
||||
@@ -29,19 +32,37 @@ Prose and diagrams only; no configuration is pasted.
|
||||
|
||||
## 1. What the bus is for
|
||||
|
||||
The bus carries five kinds of traffic today, and this design keeps the five, renaming nothing a
|
||||
module can see:
|
||||
**The bus is where the mesh happens.** Not a transport the mesh sends things over — the place a
|
||||
module is reachable at all, where a role is addressed without knowing who holds it, where the
|
||||
mesh's own state lives, and where what a module may say is decided by what it declared.
|
||||
|
||||
| Traffic | Today | Guarantee it needs |
|
||||
| Traffic | Shape | Guarantee it needs |
|
||||
|---|---|---|
|
||||
| **control** — a node's report, its heartbeat, a build's outcome, an enrolment | queues `control`, `.upgrades`, `.catchup` | nothing lost while the store restarts; retried; in order per node |
|
||||
| **declarations** — the controller tells a node what to be | queue `node.<name>` | the node gets the newest; a stale one is never applied |
|
||||
| **builds** — the controller asks the build machine to build | queue `builds` | at least once, one builder at a time |
|
||||
| **events** — a module says something happened | topic exchange `mesh.events`, keys `<module>.<event>` | delivered to every consumer that declared it; dead-lettered when it cannot be |
|
||||
| **tools** — one module or person asks another's tool a question | exchange `mesh.rpc`, per-tool service queues `serve.<module>.<tool>` | one answer, from one server, or a timeout |
|
||||
| **control** — a node's report, a build's outcome, an enrolment | job | nothing lost while the store restarts; retried; in order per node |
|
||||
| **heartbeat** — a node saying it is alive | fire and forget | none; a lost one is the next one |
|
||||
| **declarations** — the controller tells a node what to be | state | the node gets the newest; a stale one is never applied |
|
||||
| **builds** — work for the build machine | job | at least once, one worker at a time |
|
||||
| **events** — a module says something happened | 1:many | delivered to every consumer that declared it; dead-lettered when it cannot be |
|
||||
| **tools** — a module or a person asks another's tool | request/reply | one answer, from one server, or a timeout |
|
||||
| **work to a role** — a module submits to a capability without knowing who provides it | job | exactly one holder does it; it queues while nobody does |
|
||||
|
||||
The sdk's contract — `request`, `handle`, `publish`, `subscribe`, `close` — is the whole surface a
|
||||
module sees, and it does not change ([ADR 0039](../../02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md)).
|
||||
The last two rows are the ones worth dwelling on, because they are not messaging in the sense of
|
||||
carrying bytes from A to B. **A role is addressable**, so a caller names the capability and never
|
||||
the module or the node — and the implementation can be replaced under it without a caller
|
||||
changing ([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md)). That is a
|
||||
property of the mesh's architecture that happens to be expressed in subjects.
|
||||
|
||||
And more of the mesh lands here as it is built: conditions and observed state in key-value
|
||||
buckets that anything may watch, the server's own advisories becoming observations like any other
|
||||
([research 017](../../01-RESEARCH/017-a-mesh-that-heals-itself/00-overview.md)), and a person's
|
||||
client speaking the bus directly rather than through a surface built over it (§7). None of that
|
||||
is a message being moved; all of it is the bus being the mesh's centre.
|
||||
|
||||
**What a module sees of it is small and derived.** It declares what it emits, consumes, serves
|
||||
and uses, and the subjects, streams, consumers and permissions all follow from that
|
||||
([design 29](32-what-a-module-declares.md)). The sdk's contract — `request`, `handle`, `publish`,
|
||||
`subscribe`, `close` — is the whole surface, and it does not change
|
||||
([ADR 0039](../../02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md)).
|
||||
|
||||
## 2. Subjects
|
||||
|
||||
@@ -52,14 +73,30 @@ permissions are expressed as which branches of it that account may publish to an
|
||||
mesh.control.<node>.report a node's report (JetStream: CONTROL)
|
||||
mesh.control.<node>.alive heartbeat (core, no persistence)
|
||||
mesh.control.enrol an enrolment request (JetStream: CONTROL)
|
||||
mesh.control.built a build's outcome (JetStream: CONTROL)
|
||||
mesh.node.<node>.declare a declaration for a node (JetStream: NODES, last-per-subject)
|
||||
mesh.build.request work for the build machine (JetStream: BUILDS, work queue)
|
||||
mesh.events.<module>.<event> an event (JetStream: EVENTS)
|
||||
mesh.tools.<module>.<tool> a tool invocation (core request/reply)
|
||||
mesh.mod.<module>.event.<event> an event (JetStream: EVENTS)
|
||||
mesh.mod.<module>.tool.<tool> a tool invocation (core request/reply)
|
||||
mesh.seat.<seat>.accept.<verb> work submitted to a role (JetStream: per-seat work queue)
|
||||
mesh.seat.<seat>.event.<verb> a role's own event (JetStream: EVENTS)
|
||||
mesh.seat.<seat>.tool.<verb> a role's tool (core request/reply)
|
||||
mesh.ask.<node>.<command> the controller's command api (core request/reply)
|
||||
```
|
||||
|
||||
**Revised 2026-09-27** ([ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md)):
|
||||
**`mesh.build.request`, `mesh.control.built` and the BUILDS stream are gone.** A build is work submitted to a role, and the
|
||||
mesh already has a shape for that — a seat's `accept` subjects, on a work queue with a queue group of
|
||||
holders, which is what a build queue shared by several machines *is*. Keeping a second mechanism for it
|
||||
meant two things to reason about and two places for a permission to be wrong. A build's outcome is the
|
||||
seat's own event — which is why the control branch loses its copy too: one publish reaches whoever
|
||||
asked, the controller that records it and the catalogue that places it — the fan-out a shared exchange gave for free, as a derived subject rather than a
|
||||
configured topology.
|
||||
|
||||
**Revised 2026-09-26** ([design 29](32-what-a-module-declares.md)): a module's events and tools
|
||||
moved from `mesh.events.*` / `mesh.tools.*` into one namespace per module, `mesh.mod.<module>.>`,
|
||||
so a module's authority over its own name is a single subject pattern the server enforces — and
|
||||
each carries a **kind token**, without which an events stream's filter would capture tool calls.
|
||||
Seats are the same shape, one namespace per role.
|
||||
|
||||
Two things this buys over the exchanges: **request/reply is native** — a tool call is one
|
||||
`request` on `mesh.tools.<module>.<tool>` answered by whichever runtime serves it (a queue group per
|
||||
tool, so several nodes may serve one tool); and **a declaration is last-per-subject** — the NODES
|
||||
@@ -69,7 +106,14 @@ exactly the current declaration and nothing older. That is the wire-level answer
|
||||
*is* the order, and a node that sees sequence n refuses n−1 by construction.
|
||||
|
||||
**A reply-to travelling through a JetStream stream is carried in the payload, never in the
|
||||
transport `Reply` field.** Revision, first review: core NATS request/reply sets the requester's
|
||||
transport `Reply` field.** *Verified against a running server, 2026-09-27*: a caller published
|
||||
asking for a reply to `_INBOX.LCr3M83q…`, and the consumer saw a `Reply` field of
|
||||
`$JS.ACK.PROBE.probe_consumer.1.1.1…`. The address is replaced, not merely at risk — so the
|
||||
payload-borne reply subject below is necessary rather than defensive, and the check is a test
|
||||
rather than a note, because a future server that stopped doing this would leave enrolment
|
||||
working and the reason for the field quietly becoming folklore.
|
||||
|
||||
Revision, first review: core NATS request/reply sets the requester's
|
||||
ephemeral inbox as the message's `Reply` field, and a plain responder answers it directly — but a
|
||||
message a JetStream consumer delivers has already had that field claimed for the consumer's own
|
||||
ack address (`$JS.ACK.<stream>.<consumer>...`), so by the time the controller (§3's CONTROL
|
||||
@@ -89,10 +133,9 @@ Core NATS is at-most-once. Everything the mesh must not lose lives in a JetStrea
|
||||
|
||||
| Stream | Subjects | Retention | Why |
|
||||
|---|---|---|---|
|
||||
| CONTROL | `mesh.control.>` except `alive` | work queue, one consumer (the controller), explicit ack | the store-window guarantee ([ADR 0083](../../02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md)): the controller `nak`s with a delay while its store is away and the message is redelivered; nothing is dropped |
|
||||
| CONTROL | `mesh.control.>` except `alive` (a build's outcome moved to its seat, ADR 0121) | work queue, one consumer (the controller), explicit ack | the store-window guarantee ([ADR 0083](../../02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md)): the controller `nak`s with a delay while its store is away and the message is redelivered; nothing is dropped |
|
||||
| NODES | `mesh.node.>` | last per subject | one declaration per node, always the newest |
|
||||
| BUILDS | `mesh.build.>` | work queue, explicit ack | at least once; a builder that dies mid-build has its message redelivered |
|
||||
| EVENTS | `mesh.events.>` | limits (age, size), durable consumer per subscribing module | a subscriber that was down catches up; after `max-deliver` attempts the advisory feeds `mesh.events.dead` (its own small stream) |
|
||||
| EVENTS | `mesh.mod.*.event.>` | limits (age, size), durable consumer per subscribing module | a subscriber that was down catches up; after `max-deliver` attempts the advisory feeds `mesh.events.dead` (its own small stream) |
|
||||
|
||||
Tool calls and heartbeats stay on core NATS: a lost heartbeat is the next heartbeat; a lost tool
|
||||
call is a timeout the caller already handles.
|
||||
@@ -100,6 +143,33 @@ call is a timeout the caller already handles.
|
||||
Streams and consumers are objects the controller creates at genesis and asserts on start; a module
|
||||
declares nothing about them. The controller is the only writer of stream definitions.
|
||||
|
||||
### The store window, and what moving it into the server changes
|
||||
|
||||
The guarantee ([ADR 0083](../../02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md)) is
|
||||
that a push the controller cannot record because its store is restarting is **held and retried** —
|
||||
never dropped, never falsely acknowledged. Here that is a `nak` with a delay: the server holds
|
||||
the message and redelivers it, so the controller keeps no list of parked messages and one that
|
||||
restarts mid-window loses nothing it was holding.
|
||||
|
||||
**That is a plain win, and it introduces one problem worth naming.** Holding a delivery in memory
|
||||
let the controller drop an older report when a newer one for the same node arrived, because
|
||||
acting on the older after the newer would undo the newer. A `nak`ed message belongs to the server
|
||||
and comes back whatever happened meanwhile — so the older report is redelivered *after* the newer
|
||||
was applied.
|
||||
|
||||
The answer was already in the message. A report carries the **digest of the declaration it is
|
||||
about**, which exists because an earlier attempt to order reports by time lost the race it
|
||||
invited: an apply that began under the previous declaration finishes after the next is sent, and
|
||||
its report reads as newer than the send. Clocks cannot answer *which*.
|
||||
|
||||
So supersession stops being something the controller remembers and becomes something it checks —
|
||||
a report whose digest is not the one outstanding for that node is acknowledged without being
|
||||
acted on. The same shape as a node refusing a superseded declaration by sequence
|
||||
([issue 107](../../04-ISSUES/107-a-declaration-carries-no-order/00-report.md)): **ordering settled
|
||||
by what a message says, not by when it arrived.** And staleness is checked before the store is
|
||||
waited on, so a redelivery that lost its race does not hold a slot in the window that a current
|
||||
message needs.
|
||||
|
||||
## 4. Accounts
|
||||
|
||||
[ADR 0043](../../02-DECISIONS/0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md) says a
|
||||
@@ -128,19 +198,56 @@ expresses this exactly, per subject, and better than a vhost could:
|
||||
permissions name only that one prefix, for the reply to any request it makes and nothing
|
||||
wider. First review found the account note without this and read it as "any user may
|
||||
subscribe any inbox" — which was accurate against the text as it stood.
|
||||
|
||||
**And a scoped inbox needs `allow_responses`, or nothing can answer.** Revision, found while
|
||||
composing the first real configuration: the rule above scopes each user's inbox to itself,
|
||||
which is right — and leaves a responder unable to reply, because the answer goes to the
|
||||
*caller's* inbox, which the responder has no permission for. The two ways out are granting
|
||||
every responder `_INBOX.>`, which is exactly the blanket grant this bullet refuses, or NATS's
|
||||
own `allow_responses`: the server permits one reply to the reply-subject of a message the user
|
||||
actually received, within a TTL, and nothing else. So authority to answer is bounded by having
|
||||
been asked, and only principals that serve something are granted it — a pure consumer gets
|
||||
nothing. Without this the scoping is not merely incomplete: every tool call in the mesh times
|
||||
out, and the permission list looks correct while it happens.
|
||||
- **One user per module per node**, as today, with publish permissions
|
||||
`mesh.events.<module>.<event>` for each emit, `mesh.tools.<module>.>` to serve its tools, its
|
||||
own ack-reply subject for each durable consumer it holds, and its own inbox prefix; subscribe
|
||||
permissions for each consumed event's subject, its tool subjects, and that same inbox prefix.
|
||||
Nothing else. A module that tries to publish outside its emits is refused by the server, not by
|
||||
convention.
|
||||
- **The controller's user** owns `mesh.control.>`, `mesh.node.>`, `mesh.build.>` and the streams.
|
||||
- **The controller's user** owns `mesh.control.>`, `mesh.node.>` and the streams, and may submit work
|
||||
to the seats the mesh's own flows use — a build, for one (ADR 0121).
|
||||
**A host's user** may publish its own `mesh.control.<node>.>` and subscribe its own
|
||||
`mesh.node.<node>.declare` — and nothing of any other node's.
|
||||
- **A person's user** (§7) is a module-shaped user with permissions on the tool subjects it may
|
||||
invoke, issued and revoked by the controller like any account.
|
||||
|
||||
**Accounts are configuration, not API calls.** The controller composes the server's user list and
|
||||
**The server does not verify client certificates, and TLS is still required.** *Revision,
|
||||
2026-09-27, found by building the module's image and connecting to it as a host would.* The first
|
||||
composed configuration said `verify: true`, which makes the server demand a **client** certificate —
|
||||
and nothing in the mesh presents one. A host pins this server's exact certificate and authenticates
|
||||
with the password the mesh minted ([ADR 0004](../../02-DECISIONS/0004-a-node-and-how-it-joins.md)),
|
||||
and a module's runtime does the same. With it on, every connection in the mesh dies at the TLS
|
||||
handshake before any password is looked at, and the error — "client didn't provide a certificate" —
|
||||
reads as a fault in the client rather than in the bus's configuration. The `tls` block is what makes
|
||||
TLS required; `verify` only decides whether client certificates are checked. What is given up is a
|
||||
second factor the mesh has no machinery to issue or rotate — a certificate per module per node — and
|
||||
what is kept is stronger than a name check in both directions: an exact pin outward, a password
|
||||
scoped per user inward. **Mutual TLS is a later question and would need that machinery first.**
|
||||
|
||||
**The mesh composes the accounts; the module composes its server.** *Revision, 2026-09-27, while
|
||||
building the composition.* An earlier reading of the paragraph below had the controller writing the
|
||||
whole file. It writes only the user list. A server's ports, its TLS paths and its store directory
|
||||
are properties of the container the module raises — they live in its image and its mounts and change
|
||||
when it does — so the module declares its own configuration and `include`s the mesh's half. A
|
||||
controller that wrote the whole file would have to be kept in step with a Dockerfile it never sees,
|
||||
and a module could not change its own image without the mesh agreeing. Asking for the user list is
|
||||
not enough to receive it: the file holds every user's password hash, so the claim on `mesh-broker` is
|
||||
what authorises it. And the two files share one directory of necessity — an absolute include path is
|
||||
resolved relative to the including file's own directory, so a server given one from elsewhere looks
|
||||
for it underneath that directory and refuses to start.
|
||||
|
||||
**Accounts are configuration, not API calls.** The controller composes the mesh's user list and its
|
||||
permissions into a file the host declares. **How that file reaches the running server is §5's,
|
||||
not this one's** — revision, first review: an earlier draft said "reloads" and cited a precedent
|
||||
that does not apply to a container (see §5). No management API, no credential travelling through a
|
||||
@@ -157,9 +264,14 @@ signing hierarchy for nothing.
|
||||
|
||||
`nats` is a catalogue module claiming the seat `mesh-broker`
|
||||
([ADR 0079](../../02-DECISIONS/0079-the-foundation-seats-are-named-after-their-servers.md): the seat
|
||||
is the server, and the server changes). It declares one container (a single binary; JetStream on a
|
||||
named volume), its listening ports — client, TLS, and the monitoring endpoint on loopback — and a
|
||||
configuration file the controller composes (accounts, permissions, TLS, JetStream).
|
||||
is the server, and the server changes). It declares one container (a single binary; **JetStream on a host
|
||||
directory bind, not a named volume** — revision, second review:
|
||||
[issue 115](../../04-ISSUES/115-a-named-docker-volume-is-invisible-and-one-flag-from-gone/00-report.md)
|
||||
is resolved, and converted the store, the broker and two others away from named volumes for the
|
||||
reason it names; the bus's own data is not the place to reintroduce one), its listening ports —
|
||||
**the client port, which carries TLS itself** rather than standing beside a plaintext one as the
|
||||
AMQP broker's 5671/5672 pair did, and the monitoring endpoint on loopback — and a configuration
|
||||
file the controller composes (accounts, permissions, TLS, JetStream).
|
||||
|
||||
**How that file's changes reach the running server, corrected on revision.** First review: the
|
||||
earlier draft named `reload-on` as the mechanism, citing the container runtime's own trust file as
|
||||
@@ -186,10 +298,33 @@ same place `modules/gitea/token.ts` keeps its own state rather than asking the h
|
||||
The host's only job is what it already does for any directory resource: keep the file's content
|
||||
current. Nothing is declared as `reload-on` or `restart-on` for this resource at all.
|
||||
|
||||
Its guard is the same rule as the AMQP broker's: the monitoring port is refused from anything but
|
||||
Its guard is the same rule as the deprecated broker's: the monitoring port is refused from anything but
|
||||
the private network. It is raised at genesis like the store, adopted as a module in the same
|
||||
phase. The predecessor's AMQP broker remains a module of its own, `lavinmq-compat`, with a single
|
||||
purpose and a retirement condition: no client connected for a period the operator sets.
|
||||
phase.
|
||||
|
||||
**The deprecated broker is an ordinary module, not a compatibility layer.** Revision, second review
|
||||
([ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md): AMQP is not a provision, and everything speaks to the `mesh-broker` seat)): earlier text here, and
|
||||
ADR 0106 before it, called it `lavinmq-compat` — one purpose, the predecessor's clients, and a
|
||||
retirement condition of no client connected for a period the operator sets. It is none of those.
|
||||
A module may legitimately need an AMQP broker as a **backing service**, the way it needs a
|
||||
database, and the provider that answers that is an ordinary module like any other: no seat, not
|
||||
foundation, never raised at genesis, installed when something wants it and absent from a mesh
|
||||
that does not. There is no retirement condition, because the day its last client disappears is
|
||||
not a day anything is waiting for.
|
||||
|
||||
**Revised 2026-09-27** ([ADR 0130](../../02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md)):
|
||||
**that day is coming.** The predecessor is deprecated — some of it still running, none of it being
|
||||
migrated, left to stop rather than moved — so the broker retires once nothing requires `amqp`. Still no
|
||||
retirement *condition* and no end-date machinery: a provision with no consumers has its provider
|
||||
unassigned, which is the ordinary mechanism and is ADR 0127 being paid off rather than revised. What
|
||||
also goes with it is the tooling that reaches this installation's machines remotely, because the
|
||||
predecessor's own mesh talks over that broker — so the rollout is driven from the node, or before the
|
||||
broker stops.
|
||||
|
||||
What is deprecated is AMQP as **the mesh's transport**, which is this whole document. The rule
|
||||
that remains is about direction rather than software: *inter-module communication goes over the
|
||||
bus.* A module may hold a broker, a database or a cache for itself; it may not use one as a
|
||||
channel to another module.
|
||||
|
||||
## 6. Joining: the enrolment handshake
|
||||
|
||||
@@ -272,7 +407,7 @@ wrong until there is a second mesh. *Ends at: the genesis bed.*
|
||||
**Step 2 — adoption puts the broker in its seat.** A mesh already running does not get a foundation
|
||||
module by being raised again; it adopts one in place
|
||||
([ADR 0100](../../02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md)). The
|
||||
server is raised beside the AMQP broker on its own ports, carrying no mesh traffic yet, and the
|
||||
server is raised beside the deprecated broker on its own ports, carrying no mesh traffic yet, and the
|
||||
`nats` module is adopted onto it. The seat it claims is **`mesh-broker`**, unchanged — the
|
||||
foundation seats are named after the server's role rather than the product
|
||||
([ADR 0079](../../02-DECISIONS/0079-the-foundation-seats-are-named-after-their-servers.md)) for
|
||||
@@ -333,7 +468,7 @@ itself enforces and a plain client can therefore check:
|
||||
watcher-poll interval, without a restart.
|
||||
|
||||
**Step 2 — the adoption bed**, a mesh already running that has never had this server:
|
||||
- the server is raised beside the AMQP broker on its own ports and the `nats` module is adopted
|
||||
- the server is raised beside the deprecated broker on its own ports and the `nats` module is adopted
|
||||
onto it in place, holding the data and the configuration it was raised with;
|
||||
- the seat it claims is `mesh-broker`, and a second assignment of it anywhere in the mesh is
|
||||
refused at resolution — *one per mesh*, as ADR 0079 requires;
|
||||
@@ -398,8 +533,11 @@ find what changed and why.
|
||||
|
||||
**Still open:**
|
||||
|
||||
- Whether EVENTS should be one stream or one per emitting module (retention per module vs. one
|
||||
policy). One stream is proposed; the review may disagree.
|
||||
- ~~Whether EVENTS should be one stream or one per emitting module.~~ **Closed**
|
||||
([ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md): AMQP is not a provision, and everything speaks to the `mesh-broker` seat) (superseding [ADR 0125](../../02-DECISIONS/0125-the-bus-is-the-only-broker.md))): one stream, and not as a
|
||||
preference — the streams the bus is made of are composed as configuration before any module
|
||||
runs, because a provisioner is itself a module that needs a bus account to start. Bootstrapping
|
||||
decides it.
|
||||
- The heartbeat interval and the controller's "quiet" threshold on core NATS without persistence —
|
||||
the same numbers as today are proposed.
|
||||
- Whether the person's client is a catalogue module (runs on an enrolled workstation node) or a
|
||||
|
||||
@@ -4,13 +4,17 @@ status: implemented
|
||||
code:
|
||||
- mesh-controller internal/catalogue/seats.go
|
||||
- mesh-controller internal/catalogue/resolve.go
|
||||
- mesh-controller internal/inventory/seats.go
|
||||
- mesh-controller internal/inventory/migrations/0039-a-seat-is-held-by-one-assignment-on-record.sql
|
||||
- mesh-controller cmd/mesh-controller/seats.go
|
||||
- mesh-controller cmd/mesh-controller/source.go
|
||||
- mesh-controller internal/inventory/migrations/0032-a-source-may-live-on-a-seat.sql
|
||||
- mesh-catalog modules/gitea/module.json
|
||||
updated: 2026-09-27
|
||||
decisions:
|
||||
- 02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
- 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
- 02-DECISIONS/0126-a-module-declares-its-own-seats.md
|
||||
- 02-DECISIONS/0128-the-mesh-bus-is-required-not-ambient.md
|
||||
- 02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md
|
||||
- 02-DECISIONS/0109-a-package-registry-seat-is-one-per-ecosystem.md
|
||||
- 02-DECISIONS/0117-a-machines-uplink-is-a-seat.md
|
||||
@@ -41,6 +45,31 @@ is refused. A seat makes a role singular, never a module.
|
||||
**The seat points at the assignment.** Everything the mesh knows about the holder is what it knows
|
||||
about that assignment: the node, the node's settings for the module, and what the module serves.
|
||||
|
||||
**Which assignment holds a seat is a fact on record, and changes as one act.** Revision, 2026-09-27
|
||||
([ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)). Until
|
||||
then the holder was derived — the module that is assigned and claims the seat holds it, and a second
|
||||
eligible assignment was refused. That has no way to pass a seat from one holder to the next without a
|
||||
moment in which nobody holds it, and the controller finds its own bus through one of these seats: that
|
||||
moment took the control plane down for an evening. So the holder is now one row the controller keeps,
|
||||
written by a handover — `seat <name> --to <node>/<module>` — that names the seat and the assignment
|
||||
taking it over and replaces the previous holder in the same write. Between two handovers the seat has
|
||||
exactly one holder, and it is never none.
|
||||
|
||||
Three consequences follow. **A seat with no row is held as it always was**: the sole eligible
|
||||
assignment holds it, and two eligible ones are refused — so a mesh that has never handed a seat over
|
||||
behaves exactly as before, and the row appears the first time somebody does. **With a row, any other
|
||||
assignment whose module could hold the seat is eligible and silent**: neither refused nor holding.
|
||||
That is what lets the next holder run beside the current one until the handover, which the bus's move
|
||||
needs ([28](28-building-the-bus.md), task 5.3). **And a holding is the assignment's**: unassigning the
|
||||
holder takes the row with it, so a seat never points at something that is not running anywhere, and
|
||||
the seat falls back to derivation rather than to nothing.
|
||||
|
||||
The handover refuses what would make the new holder wrong before anything is written: the seat must
|
||||
exist, the assignment must exist, and the module must be able to hold the seat — claim it at its scope
|
||||
and provide what it delivers, judged against the store's row and not against anything compiled into a
|
||||
binary. It does not check that the module is running yet; `push` confirms that afterwards, and a
|
||||
handover that could only be recorded after the new holder was up could not be the switch.
|
||||
|
||||
**The set is closed.** A seat the mesh does not define is refused wherever it is named, and so is one
|
||||
named at the wrong scope. Adding a seat is a decision, recorded, for the reason every addition to the
|
||||
host's vocabulary is one: the set is what a person reads to learn what a mesh can have, and an entry
|
||||
@@ -48,28 +77,55 @@ nobody argued for is an entry nobody can explain.
|
||||
|
||||
## The set
|
||||
|
||||
| seat | scope | delivers | typically held by |
|
||||
|---|---|---|---|
|
||||
| `mesh-controller` | mesh | — | the controller |
|
||||
| `mesh-store` | mesh | — | the store the mesh's own records live in |
|
||||
| `mesh-broker` | mesh | — | the broker carrying the mesh's own bus |
|
||||
| `mesh-vault` | mesh | `secret`, reserved | the vault |
|
||||
| `the-artifact-store` | mesh | `artifact-store` | the artifact registry |
|
||||
| `the-catalogue` | mesh | — | the catalogue |
|
||||
| `npm-package-registry` | mesh | `npm-package-registry` | the forge |
|
||||
| `git` | mesh | `git` | the forge |
|
||||
| `the-build-machine` | node | — | a builder |
|
||||
| `the-dns-port` | node | — | the local resolver |
|
||||
| `the-intrusion-prevention` | node | — | an intrusion-prevention service |
|
||||
| `the-packet-filter` | node | — | the packet filter |
|
||||
| `the-private-network` | node | — | the private network the mesh runs over |
|
||||
| `the-resolver-configuration` | node | — | whichever of the alternative resolver configurations is chosen |
|
||||
| `the-showcase` | node | — | the showcase module |
|
||||
| `the-uplink` | node | — | the program that manages the machine's own network ([ADR 0117](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md)) |
|
||||
**The set is derived, and only the mesh's half is written here.** Revision, 2026-09-26
|
||||
([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md), superseding
|
||||
[ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)): a module
|
||||
declares its own seats with their protocols, so the seats a mesh has are the mesh's own **plus
|
||||
every registered module's**. The set is still closed — a seat named nowhere is refused — but it is
|
||||
computed from the catalogue rather than maintained by hand, which is the property 0110 actually
|
||||
needed and the table could not keep.
|
||||
|
||||
The controller holds this set in code, and a test asserts both its size and that every entry names
|
||||
the record that made it a seat. **This table and [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)
|
||||
govern, and code that disagrees is what is wrong.** The implementation in progress predates several
|
||||
**And the mesh's own half is data, named for its scope.** Revision, 2026-09-27, reconciling two
|
||||
records made in parallel: [ADR 0121](../../02-DECISIONS/0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md)
|
||||
names a system seat for the scope it is held at — `mesh-*` for one per mesh, `node-*` for one per
|
||||
machine — and [ADR 0122](../../02-DECISIONS/0122-a-seat-is-data-a-rename-is-a-database-update.md) moves
|
||||
the set out of compiled code into a table the controller owns, so a rename is one write rather than a
|
||||
rebuild of everything that names one.
|
||||
|
||||
So the set has two halves and neither is written out here: the mesh's own, which the controller holds
|
||||
as rows, and every registered module's, which is computed from the catalogue. What this document keeps
|
||||
is what a seat *is* — the rest would be a third copy, stale the first time somebody renamed one, which
|
||||
is the fault ADR 0122 exists about.
|
||||
|
||||
|
||||
**Every seat below is named `mesh-*`, and the prefix is the reservation rule**: a module declaring
|
||||
any `mesh-*` name is refused at registration, so there is no reserved-names list to drift. Ten of
|
||||
these are renamed to restore [ADR 0079](../../02-DECISIONS/0079-the-foundation-seats-are-named-after-their-servers.md)'s
|
||||
convention, which later seats departed from.
|
||||
|
||||
| seat | was | scope | delivers | typically held by |
|
||||
|---|---|---|---|
|
||||
| `mesh-controller` | — | mesh | — | the controller |
|
||||
| `mesh-store` | — | mesh | — | the store the mesh's own records live in |
|
||||
| `mesh-broker` | — | mesh | `mesh-bus` | the broker carrying the mesh's own bus |
|
||||
| `mesh-vault` | — | mesh | `secret`, reserved | the vault |
|
||||
| `mesh-artifact-store` | `the-artifact-store` | mesh | `artifact-store` | the artifact registry |
|
||||
| `mesh-catalog` | `the-catalogue` | mesh | — | the catalogue |
|
||||
| `mesh-npm-package-registry` | `npm-package-registry` | mesh | `npm-package-registry` | the forge |
|
||||
| `mesh-git` | `git` | mesh | `git` | the forge |
|
||||
| `mesh-build-machine` | `the-build-machine` | node | — | a builder |
|
||||
| `mesh-dns-port` | `the-dns-port` | node | — | the local resolver |
|
||||
| `mesh-intrusion-prevention` | `the-intrusion-prevention` | node | — | an intrusion-prevention service |
|
||||
| `mesh-packet-filter` | `the-packet-filter` | node | — | the packet filter |
|
||||
| `mesh-private-network` | `the-private-network` | node | — | the private network the mesh runs over |
|
||||
| `mesh-resolver-configuration` | `the-resolver-configuration` | node | — | whichever of the alternative resolver configurations is chosen |
|
||||
| `mesh-showcase` | `the-showcase` | node | — | the showcase module |
|
||||
|
||||
The controller holds **the mesh's own** entries in code, and a test asserts their size and that
|
||||
every one names the record that made it a seat. A module's seats are not here and never will be —
|
||||
they are read from the catalogue. **This table and
|
||||
[ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) govern, and code that
|
||||
disagrees is what is wrong.** The implementation in progress predates several
|
||||
things here: seats held by assignments rather than claimed by definitions, the `mesh-vault` seat and
|
||||
its reservation, and the foundation's seats delivering nothing. It is brought to this table before it
|
||||
merges.
|
||||
@@ -121,6 +177,15 @@ moves that to a host port requirement.
|
||||
|
||||
## A seat that delivers nothing
|
||||
|
||||
**A module's declared seat may promise nothing too, and that is a marker seat.** Correction of fact,
|
||||
2026-09-27: [ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md)'s "a declared seat
|
||||
carries a protocol" governs what a *holder* must satisfy, not that every seat offers something. A
|
||||
marker seat's protocol is satisfied by holding it, which is the whole of what a marker says. Refusing
|
||||
an empty one would refuse most of the node-scoped set, the showcase module's own seat included.
|
||||
Nothing reaches that state by accident: an unknown manifest field is refused outright, so an empty
|
||||
protocol was written as one. Checked by a registration test accepting a node seat with no protocol
|
||||
and by the showcase manifest, which declares one.
|
||||
|
||||
Most node seats deliver nothing. They say which module is this machine's packet filter, or which of
|
||||
two alternative resolver configurations it runs, and a second holder is refused. That is the whole of
|
||||
their job, and it is a real one: it is the mesh saying what a machine is, in words a person can read.
|
||||
@@ -157,16 +222,24 @@ still to take.
|
||||
|
||||
## How it is checked
|
||||
|
||||
The rules here are [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)'s
|
||||
and [ADR 0111](../../02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md)'s, and each is
|
||||
The rules here are [ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md)'s —
|
||||
which supersedes [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)
|
||||
and keeps every rule below except how the set is formed — and
|
||||
[ADR 0111](../../02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md)'s. Each is
|
||||
checked as their tables say:
|
||||
|
||||
| Rule | Checked by |
|
||||
|---|---|
|
||||
| The set is closed, and every entry names its decision | 0110: a unit test on the set's size and decisions; manifest tests refusing an unknown seat or the wrong scope. |
|
||||
| A seat is held by one assignment, and only by one whose module can hold it | 0110: resolution tests for a second holder and for a seat the definition does not name. |
|
||||
| A requirement naming a seat is answered by its holder; a foundation seat cannot be named | 0110: resolution tests with a second provider on the consumer's node, with the seat unheld, and naming `mesh-store`. |
|
||||
| Several providers and none local is a person's choice | 0110: an assignment test listing candidates with the seat's holder first and recording the pin. |
|
||||
| `secret` is reserved | 0110: the parser and resolution refusals for another provider and a pin. |
|
||||
| Holdings are derived, and the overview lists every seat | 0110: the `seats` command test, including an unheld seat. |
|
||||
| The set is closed, and every mesh entry names its decision | 0118: a unit test on the mesh's own entries. **The refusal of an unknown seat is at registration, not in the parser** — correction of fact, 2026-09-27: a module may hold a seat *another* module declares, which is the point of naming the seat and not its provider, so whether a claimed name exists is a fact about the whole catalogue and a manifest in isolation cannot be judged on it. Registration tests cover an invented name and a name another module declares; the parser still refuses a claim on the mesh's own `mesh-*`/`node-*` namespace and a scope that disagrees with a declaration in the same manifest. |
|
||||
| The set is derived, and enumerating it is a query | 0118: the overview lists the mesh's own plus every registered module's, asserted against a fixture mesh. |
|
||||
| `mesh-*` is the mesh's, and a module may not declare one | 0118: a registration test refusing a manifest that declares any `mesh-*` seat, naming the prefix. |
|
||||
| Two modules cannot declare the same seat | 0118: a registration test; the second is refused and the first untouched. |
|
||||
| A holder satisfies the seat's protocol | 0118: a claim whose module does not serve what the seat declares is refused at assignment. |
|
||||
| A seat is held by one assignment, and only by one whose module can hold it | 0118: resolution tests for a second holder and for a seat the definition does not name. 0131: `CanHold` is the one judgement, shared by registration and the handover, and its test follows the store's row. |
|
||||
| A holder on record settles the seat; another eligible assignment is silent, not refused | 0131: resolution tests with a recorded holder on the same machine, on another machine, and under a seat's former name; without a record, the old rule's tests still pass unchanged. |
|
||||
| A handover replaces the holder as one write, needs an assignment to point at, and goes with it | 0131: store tests — a second handover leaves one row; a handover to a module not assigned where named is refused; unassigning the holder removes the row. |
|
||||
| A requirement naming a seat is answered by its holder; a foundation seat cannot be named | 0118: resolution tests with a second provider on the consumer's node, with the seat unheld, and naming `mesh-store`. |
|
||||
| Several providers and none local is a person's choice | 0118: an assignment test listing candidates with the seat's holder first and recording the pin. |
|
||||
| `secret` is reserved | 0118: the parser and resolution refusals for another provider and a pin. |
|
||||
| Holdings are derived, and the overview lists every seat | 0118: the `seats` command test, including an unheld seat. |
|
||||
| A build source on the seat records no address; an unheld seat refuses only self-hosted builds | 0111's tests. |
|
||||
|
||||
@@ -8,7 +8,7 @@ decisions:
|
||||
- 02-DECISIONS/0115-one-assignment-of-a-module-per-node.md
|
||||
- 02-DECISIONS/0113-the-vault-makes-every-secret.md
|
||||
- 02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md
|
||||
- 02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md
|
||||
- 02-DECISIONS/0126-a-module-declares-its-own-seats.md
|
||||
- 02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md
|
||||
- 02-DECISIONS/0084-which-provider-serves-a-consumer.md
|
||||
- 02-DECISIONS/0046-a-module-configuration-is-its-assignments-not-its-manifest.md
|
||||
@@ -64,7 +64,7 @@ Which module answers, in order:
|
||||
1. **the holder of the seat the requirement names.** A requirement may name a seat instead of leaving
|
||||
the provider open. It asks for *the mesh's* one, and the mesh answers with whichever assignment
|
||||
holds that seat, with nothing asked of anyone. Unheld, the requirement is refused, naming the seat
|
||||
([ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md),
|
||||
([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)),
|
||||
[26 — The seats](26-the-seats.md)). Only a seat that delivers a provision can be named; naming a
|
||||
foundation seat is refused, because it delivers nothing. A `secret` requirement always names
|
||||
`mesh-vault`, because that provision is reserved;
|
||||
@@ -199,7 +199,7 @@ containers, login, broker account and settings are keyed by it, as today, and a
|
||||
tightest backend ([ADR 0049](../../02-DECISIONS/0049-a-consumers-identity-fits-the-tightest-backend.md)).
|
||||
|
||||
**A module may run on many nodes, and one assignment may hold a seat**
|
||||
([ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)). The definition
|
||||
([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md))). The definition
|
||||
says which seats the module can hold; the assignment says which it does. So the store module can run
|
||||
on every node, one of those assignments holds `mesh-store`, and moving that role changes an
|
||||
assignment, not a definition.
|
||||
|
||||
@@ -1,15 +1,21 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: proposed
|
||||
code: []
|
||||
updated: 2026-09-26
|
||||
status: in-progress
|
||||
code:
|
||||
- mesh-catalog modules/nats
|
||||
- mesh-controller internal/catalogue
|
||||
- mesh-lab scenarios
|
||||
updated: 2026-09-28
|
||||
decisions:
|
||||
- 02-DECISIONS/0116-the-bus-is-built-in-five-steps.md
|
||||
- 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
- 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
- 02-DECISIONS/0126-a-module-declares-its-own-seats.md
|
||||
- 02-DECISIONS/0074-the-wire-is-specified-not-the-types.md
|
||||
- 02-DECISIONS/0079-the-foundation-seats-are-named-after-their-servers.md
|
||||
- 02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md
|
||||
- 02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md
|
||||
- 02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md
|
||||
---
|
||||
|
||||
# 28. Building the bus
|
||||
@@ -111,25 +117,153 @@ step 5 the rollout
|
||||
|
||||
## Step 1 — the module, and genesis raises it
|
||||
|
||||
> **Revised 2026-09-26** ([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md),
|
||||
> [design 29](32-what-a-module-declares.md)). Tasks 1.3 and 1.4 said the controller composes every
|
||||
> account and creates *the four streams* at genesis, from a fixed set. That is only the mesh's own
|
||||
> half. A module declares seats with their protocols, so streams are created **at registration**
|
||||
> and durable consumers **at assignment** — neither of which has happened at genesis. The fixed
|
||||
> foundation set stays here; the derived machinery moves to step 3, where the declaration model it
|
||||
> reads from is specified. Tasks 1.1 and 1.2, already done, are untouched by this: the module and
|
||||
> its reload mechanism do not care what the configuration says.
|
||||
|
||||
**Why here.** Everything else needs a server to talk to, and genesis is where the foundation is
|
||||
defined. The mesh this is for will never travel this path — it is already running, and takes step 2
|
||||
— but genesis is the definition every other path is measured against, and one that exists only on
|
||||
paper is wrong until there is a second mesh to find out.
|
||||
|
||||
- [ ] 1.1 the `nats` module: manifest, image, one container, its client, TLS and monitoring ports,
|
||||
- [x] 1.1 the `nats` module: manifest, image, one container, its client, TLS and monitoring ports,
|
||||
JetStream on a named volume — the shape of design 25 §5, and the same shape the broker module
|
||||
beside it already has
|
||||
- [ ] 1.2 the composed configuration as a **directory** resource, and the entrypoint that watches
|
||||
- [x] 1.2 the composed configuration as a **directory** resource, and the entrypoint that watches
|
||||
the one file and signals the server itself — design 25 §5's correction, kept inside the module
|
||||
because a container has no reload and a recreate would drop every connection the mesh has
|
||||
- [ ] 1.3 the controller composes that file: accounts, permissions, TLS, JetStream — permissions
|
||||
derived from `emits` and `consumes` and nothing else, plus each user's own ack subject and its
|
||||
own inbox prefix (design 25 §4)
|
||||
- [ ] 1.4 the four streams, created at genesis and asserted idempotently on start, by the controller
|
||||
as their only writer
|
||||
- [ ] 1.5 genesis raises it as foundation, claiming the seat **`mesh-broker`** — the seat is the
|
||||
server's role, not the product
|
||||
- [ ] 1.6 the genesis-broker bed
|
||||
- [x] 1.3 the controller composes that file: accounts, permissions, TLS, JetStream — a user's
|
||||
permissions derived from its declaration and nothing else, over the three namespaces of
|
||||
[design 29](32-what-a-module-declares.md) §2, plus its own ack subject and its own inbox
|
||||
prefix (design 25 §4)
|
||||
- [x] 1.4 the mesh's own streams, created at genesis and asserted idempotently on start, by the
|
||||
controller as their only writer — **the mesh's own, not all of them**: a seat's streams are
|
||||
created when the module declaring it is registered, and a module's durable consumers when it
|
||||
is assigned, so this task is the fixed foundation set and 3.x carries the derived rest
|
||||
- [x] 1.5 genesis raises it as foundation, claiming the seat **`mesh-broker`** — the seat is the
|
||||
server's role, not the product. **Already true of the controller and needed no change**: it
|
||||
resolves the broker by seat ("that is where the broker is, whatever else the topology says")
|
||||
and names no broker module anywhere in its source. What remains is naming `nats` instead of
|
||||
the deprecated broker where a genesis module set is declared, which is scenario and installer
|
||||
configuration — carried with 1.6 rather than before it.
|
||||
- [x] 1.7 **the composition, delivered** — the controller gathering its principals, composing the
|
||||
file, and asserting the streams and consumers on start.
|
||||
|
||||
**In**: the user list is derived from the mesh's records and the credentials are kept.
|
||||
|
||||
A bus user's bcrypt hash is now recorded, keyed by the username the file needs, and the
|
||||
plaintext is returned exactly once. That state is new and the reason is worth stating: on the
|
||||
bus the mesh runs on today an account is a management call — mint, hand over, seal to the
|
||||
holder, keep nothing — and that works because the broker remembers. Here the users are one
|
||||
file rewritten whenever any of it changes, so keeping nothing would mean **the first person's
|
||||
access change silently blanking every module's password**.
|
||||
|
||||
**Permissions are not kept, only credentials.** Authority is derived from what each module
|
||||
declares every time the file is written ([ADR 0043](../../02-DECISIONS/0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md));
|
||||
a stored permission list would be a second account of a user's authority, able to disagree
|
||||
with the records it came from while both looked internally consistent.
|
||||
|
||||
The derivation refuses two things where they can still be named: two users with one name — the
|
||||
server reads the file as one of them and which one depends on the order — and a module
|
||||
assigned but absent from the catalogue, which would compose a user with no authority and fail
|
||||
on its first publish with an authorisation error that says nothing about a missing manifest.
|
||||
A user the mesh has minted no password for is *named* rather than dropped or written as a user
|
||||
anybody is: an ordinary situation with an obvious remedy, and the caller decides whether a
|
||||
partial file is worth writing. A seat's protocol is gathered across the whole catalogue, not
|
||||
from one manifest, because a seat is declared by one module and held by another.
|
||||
|
||||
**Out, and what each needs.**
|
||||
|
||||
**Delivery is in, and it settled what a module declares.** The mesh writes the *accounts* and
|
||||
the module owns its *server*. The alternative was a manifest field enumerating ports, TLS
|
||||
paths and a store directory so the controller could write a whole configuration — wrong,
|
||||
because those are properties of the container the module raises and the controller would have
|
||||
to be kept in step with a Dockerfile it never sees. So a module declares its own configuration
|
||||
as a file resource and `bus-users` names where the mesh's half goes beside it; **asking is not
|
||||
enough to receive it**, because that file holds every user's password hash, so the claim on
|
||||
`mesh-broker` is what authorises it.
|
||||
|
||||
Two things a running server changed. **An absolute include path is resolved relative to the
|
||||
including file's directory** — `include /etc/nats/accounts.conf` from another directory makes
|
||||
the server look for it *under* that directory and refuse to start — so both files share one.
|
||||
And **`verify: true` was refusing every connection in the mesh**: it makes the server demand a
|
||||
*client* certificate, and nothing in the mesh presents one — a host pins this server's exact
|
||||
certificate and authenticates with the password the mesh minted ([ADR 0004](../../02-DECISIONS/0004-a-node-and-how-it-joins.md),
|
||||
design 25 §4). Every connection would have died at the TLS handshake before any password was
|
||||
looked at, with an error that reads as a fault in the client. Removed; TLS is still required,
|
||||
because the block is what requires it and `verify` only decides whether client certificates
|
||||
are checked. **Design 25 §4 should say this**, and says nothing about it today.
|
||||
|
||||
Also collected here: task 1.2's payoff, end to end against the module's own image — the user
|
||||
list rewritten, the module noticing and reloading the server itself with no signal from
|
||||
outside, and the connection the mesh already had still working afterwards.
|
||||
|
||||
**Minting is in, on both halves.** A node at enrolment, a module when its credential is
|
||||
issued. Three things differ from a management call and each is the point of the move: the
|
||||
credential is minted into the mesh's records and becomes usable at the next composition, so no
|
||||
server need be reachable for it; the password travels beside the address rather than inside it,
|
||||
because a credential embedded in a URL leaks into every log line that prints a connection; and
|
||||
a module's durable consumer is derived from what it declared rather than named, so it cannot ask
|
||||
for delivery of something it did not say it consumes. A node reconnecting may be refused until
|
||||
the composition reaches the machine running the bus — which is what the host's reconnect backoff
|
||||
is for, where waiting for the push would hold an enrolment open for as long as a declaration
|
||||
takes to apply.
|
||||
|
||||
**Which bus is one fact, and being told about both is refused at start.** Not warned about: a
|
||||
mesh half on each is one where a declaration goes out on one bus and the report comes back on
|
||||
the other, and every component logs success while it happens — ADR 0074's failure arriving
|
||||
through configuration instead of through code. A node that came away holding a credential for
|
||||
each could be half-moved, and nothing would say which half.
|
||||
|
||||
**The objects are asserted on every start**, not created once at genesis: a stream somebody
|
||||
deleted, a mesh raised from a restored backup, or a bus whose data directory was replaced all
|
||||
have records and no objects, and a node whose consumer is missing hears nothing while everything
|
||||
else about it looks correct. Against a real server: every object accepted, asserting twice
|
||||
changes nothing (a start that failed the second time is a controller that cannot restart), a
|
||||
machine joining an already-raised bus accepted, each node's consumer bound to its own
|
||||
declaration subject and no other's, and CONTROL not dead-lettering — because the store window's
|
||||
bound is the controller's, and a server that gave up first would discard the push the stream
|
||||
exists to protect.
|
||||
|
||||
**People are not in the list**, deliberately: the account model is built and `operator issue`
|
||||
is not (4.4), so there is nobody to derive. Left empty rather than guessed at.
|
||||
|
||||
> **This corrects a tick, not a decision.** Tasks 1.3 and 1.4 are ticked and they are honest
|
||||
> about what they built — the composer, the derivation, the permission model, the stream and
|
||||
> consumer definitions, the asserter, all pure and held by unit tests and a golden
|
||||
> composition. What nobody wrote is the *caller*. Measured on the feature branch: outside the
|
||||
> package that defines them, there is **not one** use of the composer, the permission
|
||||
> derivation, the stream set, the stream asserter or the principal type. Step 1's "done when"
|
||||
> claims "every account and permission composed from the manifests", and a mesh raised today
|
||||
> would stand up a server with no user list at all.
|
||||
>
|
||||
> It also needs state the mesh does not keep. Design 25 §4 says the file holds bcrypt
|
||||
> hashes, and passwords are "minted and sealed exactly as today" — but today the mesh mints a
|
||||
> password, hands it to the broker through a management call, seals the plaintext to the
|
||||
> holder and **keeps nothing**. There is no management call here, so the hash has to survive
|
||||
> for every later recomposition: the first thing a person's access change or a new module
|
||||
> touches is a file that must still contain every other user's password. No bcrypt hash is
|
||||
> stored anywhere in the controller today.
|
||||
>
|
||||
> Named as its own task rather than folded into 1.3 so the gap is visible: the parts of
|
||||
> step 1 exist and the mesh does not yet do any of it.
|
||||
|
||||
- [ ] 1.6 the genesis-broker bed — **deferred**: beds are run once, at the end, rather than per
|
||||
step (novox/hq design 22's rule, and the operator's instruction). Every claim step 1 makes
|
||||
is covered by a unit test or was demonstrated against the real server; what the bed adds is
|
||||
the claims that need a mesh.
|
||||
|
||||
> **Not done here, deliberately.** The controller builds a module's broker credential as an
|
||||
> `amqps://` URL and defaults a portless genesis address to 5671. Those are correct until the
|
||||
> rollout and must not move: steps 1 to 4 leave every node on AMQP
|
||||
> ([ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md)), so changing the
|
||||
> credential's shape now would break the running bus to serve a bus nothing speaks yet. They
|
||||
> change with the links, in step 3.
|
||||
|
||||
**Done when.** A mesh raised from nothing has the server standing with the streams asserted and
|
||||
every account and permission composed from the manifests; a user cannot publish outside its
|
||||
@@ -149,11 +283,35 @@ and it is what makes steps 3 and 4 safe to develop against a live mesh. A runnin
|
||||
a foundation module by being raised again; it adopts one in place
|
||||
([ADR 0100](../../02-DECISIONS/0100-a-node-in-use-is-adopted-before-it-is-converged.md)).
|
||||
|
||||
- [ ] 2.1 the server raised beside the existing broker on its own ports, carrying nothing
|
||||
- [ ] 2.2 the `nats` module adopted onto it in place, holding the data and configuration it was
|
||||
raised with
|
||||
- [ ] 2.3 the seat claim, and the resolver's refusal of a second holder mesh-wide
|
||||
- [ ] 2.4 the adoption bed
|
||||
- [ ] 2.1 the server raised beside the existing broker on its own ports, carrying nothing —
|
||||
installer-side, from the upstream image
|
||||
- [ ] 2.2 the `nats` module assigned, which **recreates the container once, deliberately** (see
|
||||
below), keeping its JetStream directory
|
||||
- [x] 2.3 the seat claim, and the resolver's refusal of a second holder mesh-wide — **already
|
||||
true and now proved**: the refusal is generic to any mesh-scoped seat, and three tests pin
|
||||
what matters for this one — a second bus anywhere is refused naming the seat, a *different*
|
||||
bus implementation is refused for the same reason (which is what lets the bus be replaced
|
||||
at all), and the deprecated broker no longer contends for it, so both run on one mesh
|
||||
- [ ] 2.4 the adoption bed — deferred with the other beds
|
||||
|
||||
> **Adoption here is not a no-op, and pretending it would be is the trap.** The host keeps an
|
||||
> existing container only when its spec matches the declaration exactly
|
||||
> ([`apply.go`](https://git.novox.be/novox/mesh-host): *existed && before.Spec == want && running*
|
||||
> → unchanged; anything else is `rm -f` and recreate). Genesis raises the server from the
|
||||
> **upstream** image, because nothing has been built yet; the module declares the **mesh-built**
|
||||
> artifact, which carries the entrypoint that reloads configuration in place. Those two specs
|
||||
> differ, so assigning the module recreates the container.
|
||||
>
|
||||
> That is correct, and it is [ADR 0067](../../02-DECISIONS/0067-genesis-is-a-pivot.md)'s pivot
|
||||
> exactly: raise a temporary thing, then reinstall it as an ordinary module. It is safe **only
|
||||
> because it happens while the bus carries nothing** — which is what 2.1 means by "carrying
|
||||
> nothing", and why step 2 comes before anything speaks NATS rather than after. One recreate, at
|
||||
> the one moment it costs nothing.
|
||||
>
|
||||
> **After that, never again.** The configuration is a directory mount rather than a file, so
|
||||
> rewriting accounts does not change the container's spec and the entrypoint reloads the server in
|
||||
> place. That is the whole point of task 1.2, and this is the moment it pays: every later account,
|
||||
> permission or person's access change touches a running bus with connections on it.
|
||||
|
||||
**Done when.** A mesh already running has the server adopted, holding `mesh-broker`; a second
|
||||
assignment anywhere is refused at resolution — *one per mesh*; and every node is still on the old
|
||||
@@ -166,19 +324,160 @@ quietly carried traffic would be step 5 arriving early and unrehearsed.
|
||||
that decides what "agreeing" means for everything after it. It is the largest step and the one that
|
||||
pays for itself furthest away.
|
||||
|
||||
- [ ] 3.1 **the suite first, on the bus the mesh has** — fixtures for the envelope and its required
|
||||
headers, the contributions file, a served tool call and a grant, capturing what the three
|
||||
implementations do *today*. Written first because a suite born on the new bus certifies
|
||||
whatever the new bus happens to do
|
||||
- [ ] 3.2 design 19 rewritten from exchanges, queues and routing keys to the subjects and streams of
|
||||
- [x] 3.1/3.3 **the fixtures** — one directory in the sdk, read by each implementation's own
|
||||
runner rather than copied into either, because a fixture copied twice is two fixtures. The
|
||||
Go emitter and the runtime's NATS client both pass the first: every required header set,
|
||||
each value in the pinned shape, the subject derived the same way, and the payload the body
|
||||
alone.
|
||||
|
||||
**The suite also had to settle what "byte-for-byte" can mean**, which ADR 0074 stated and
|
||||
nothing had yet had to implement. The envelope is exact — subject, required headers, names
|
||||
and formats — because that is what two implementations get wrong invisibly. The body is
|
||||
not: Go sorts a map's keys and JavaScript keeps insertion order, so identical bytes would
|
||||
commit every implementation to a canonical JSON encoder, to buy a property the mesh never
|
||||
uses. Read strictly it would have sent somebody writing one.
|
||||
|
||||
Still to capture: a served tool call, a grant and its answer, and the contributions file —
|
||||
the other three ADR 0074 names.
|
||||
- [x] 3.2 design 19 rewritten from exchanges, queues and routing keys to the subjects and streams of
|
||||
design 25 §2–§3, per capability, with ADR 0074's model untouched: floor plus capabilities, an
|
||||
implementation legitimate when it claims less, identity from the sealed credential, dedup on
|
||||
`x-event-id`
|
||||
- [ ] 3.3 the fixtures restated on NATS, and the capability each implementation claims
|
||||
- [ ] 3.4 the controller's link on NATS
|
||||
- [ ] 3.5 the host's link on NATS — mirroring, still importing nothing
|
||||
- [ ] 3.6 the tool runtime's client on NATS, behind the unchanged sdk contract
|
||||
- [ ] 3.7 the sdk's three stale comments, and nothing else in it
|
||||
`x-event-id`. Claims checked against a running server are marked *verified* in the text, so
|
||||
a reader can tell what was measured from what was reasoned. One limitation lifts with the
|
||||
transport: a module may now call another's tool, which issue 049 recorded it could not.
|
||||
- [x] 3.4 the controller's link on NATS — **both halves are through the seam, and the store
|
||||
window is the server's.** `Bus` states the outbound in the mesh's words (publish an event,
|
||||
declare to a node) and `Control` states the inbound (took it, dropped it, held it for the
|
||||
store); each has an AMQP and a NATS implementation, and both ship, because steps 1 to 4
|
||||
leave every node on AMQP and both shipping is what holds them to one envelope.
|
||||
|
||||
The outbound seam turned out to be eight call sites; the inbound was the larger half, and
|
||||
the reason: every handler took the transport's own delivery type, so the loop could not move
|
||||
without moving enrolment, reports, builds, upgrades and catch-up with it in one breath.
|
||||
|
||||
**The window (ADR 0083) is now what decides, once, for both.** On the bus the mesh has,
|
||||
holding a message means an unacknowledged delivery kept in the controller, bounded by the
|
||||
prefetch and lost if it stops. On the bus being built it is a `nak` with a delay: the
|
||||
message stays the server's and the controller keeps only the moment it first could not take
|
||||
it, so one that restarts mid-window has nothing to lose. Seven claims about that were asked
|
||||
of a running server rather than reasoned — a report heard and gone from the work queue, one
|
||||
held through a store outage and recorded when it returned, one let go once the bound passed,
|
||||
a superseded one settled without being acted on, a heartbeat heard and nothing persisted,
|
||||
both followed events acknowledged on a stream the controller had no ack subject for, and the
|
||||
enrolment answer arriving at the address the request carried in its payload.
|
||||
|
||||
**Three things the wiring forced into the open.**
|
||||
|
||||
*Supersession is asked before the store, not after.* A report about a declaration the mesh
|
||||
has moved past would otherwise wait out a restarting store to be written, and then overwrite
|
||||
what the node is doing now.
|
||||
|
||||
*Half of a report is not about a declaration, and that half is never stale.* What the machine
|
||||
**is** — the tunnel it took over, the ports its own bundle holds, what an adopted node found,
|
||||
a node moving its overlay key — reaches the mesh on a report and nowhere else. A rekey set
|
||||
aside as stale is a node whose overlay key never moves, and no retry is coming, because the
|
||||
node said it once. So staleness is asked only of a report that is purely an apply's account.
|
||||
|
||||
*The controller could not have consumed a module event at all.* Its account granted no event
|
||||
subject to subscribe and no ack subject on the events stream, so every announcement would
|
||||
have been redelivered for ever, refused by the permission list it already had. Both are now
|
||||
granted, each subject named rather than by pattern — a controller subscribing every event in
|
||||
the mesh is a permission list that has stopped saying what it is for. Its consumers are
|
||||
**named beside the mesh's own streams rather than derived**, because the controller files no
|
||||
manifest and authority cannot come from a declaration that does not exist.
|
||||
|
||||
Still outstanding: a build's own shape, which travels with the builder in step 4.
|
||||
- [x] 3.5 the host's link on NATS — **all three halves are through seams**, mirroring the
|
||||
controller's and still importing nothing of the mesh's own (ADR 0005): the host's own
|
||||
interfaces over its own libraries, agreeing with the controller only because a fixture holds
|
||||
both to one envelope. A report goes through JetStream because it is the message the
|
||||
store-window guarantee is about; a heartbeat stays on core, because a heartbeat in a stream is
|
||||
the mesh's least valuable message competing for retention with its most valuable.
|
||||
|
||||
`Link` is dialling, hearing and saying in one interface, because **dialling is where the
|
||||
transport is chosen** and choosing it twice is how one half of a node ends up on a different
|
||||
bus from the other. `Asking` is the enrolment conversation, and it is separate for the
|
||||
opposite reason: almost nothing about it is the same, and a node that fails there is not in
|
||||
the mesh at all.
|
||||
|
||||
**What the new bus took away, and what it would not give.** A host declares nothing here: on
|
||||
the old bus it declares its own queue, because a queue that is not there means a node that
|
||||
hears nothing, but the object it reads through now is a durable consumer and a host's account
|
||||
reaches no part of the JetStream API. So it **binds** to one the mesh made, and a missing one
|
||||
is said as the mesh's to answer rather than quietly created with whatever the client defaults
|
||||
to. Two things that had to be built for that: a node's declaration consumer (named after the
|
||||
node, because its ack grant is derived from the node's name, so any other name is a delivery
|
||||
it cannot acknowledge), and the enrolment user's **inbox** — design 25 §6 names it and the
|
||||
composer granted none, so an enrolling node would have published its request and waited out
|
||||
its timeout against a mesh that answered.
|
||||
|
||||
**The reply address travels in the payload, and that is now proved from both ends.** The
|
||||
controller reads it from there (3.4) and the host writes it there and waits on it, and the
|
||||
test asserts the transport's own reply field held the *consumer's ack address* by the time the
|
||||
request arrived — so a future server that stopped claiming that field fails a test rather than
|
||||
letting the reason quietly become folklore.
|
||||
|
||||
The host's **"newest wins" window narrows at the rollout rather than disappearing**, and that
|
||||
is now measured rather than predicted: three declarations pushed to an absent node leave one
|
||||
on the stream and it is the newest, so the catch-up half is the stream's — but three pushes to
|
||||
a connected node are still three deliveries, which is the half that stays.
|
||||
|
||||
**The pin turned out easier here than in the tool runtime, not harder.** The Go client takes a
|
||||
`*tls.Config`, so the same pinned configuration with the same verify callback does the work;
|
||||
the subject-alternative-name constraint recorded under 3.6 is that client's, because it takes
|
||||
PEM strings with no verify hook. A host checks the fingerprint and nothing else.
|
||||
|
||||
Nothing here composes an enrolment user per live token, and that is **1.7's**, not this
|
||||
task's: it is one input to a composition that does not happen at all yet.
|
||||
- [x] 3.6 the tool runtime's client on NATS, behind the unchanged sdk contract — round-tripped
|
||||
against a real server: a tool answered across two connections, a throwing handler reaching
|
||||
the caller as an error rather than a timeout, an event delivered once with its key, body,
|
||||
node and event id intact. Ships beside the AMQP client and is selected at the rollout,
|
||||
because steps 1 to 4 leave every node on AMQP.
|
||||
|
||||
**A constraint it surfaced, recorded where somebody issuing a certificate will look.** The
|
||||
AMQP client pinned the exact certificate and switched hostname verification off, which is
|
||||
sound because a fingerprint is stronger than a name. The NATS client exposes no equivalent
|
||||
hook — its TLS options are PEM strings with no verify callback — so the pin still happens
|
||||
before dialling and the library's own name check happens beside it. **The bus's certificate
|
||||
must carry a subject-alternative name matching the address nodes dial it by**, or the
|
||||
connection is refused by a library error rather than by anything the mesh says.
|
||||
- [x] 3.7 the sdk's three stale comments, and nothing else in it — three lines, which is the
|
||||
whole of the sdk's diff for the bus change, and the measurement that predicted it
|
||||
- [x] 3.8 **the declaration model** of [design 29](32-what-a-module-declares.md): local names
|
||||
derived to subjects, the three namespaces, permissions computed from a declaration, and a
|
||||
manifest that contains no subject. Done in the controller's composer (permissions, streams,
|
||||
consumers), in the runtime's client (subjects derived from the credential, never named by a
|
||||
module), and as a catalogue test asserting all 72 manifests hold no subject — because the
|
||||
rule held by construction, and a rule held by construction is one a later field breaks
|
||||
quietly.
|
||||
- [x] 3.9 **seats declared by modules** — the manifest now carries `seats` (name, scope,
|
||||
accepts/emits/serves, retention) and `uses`, and registration refuses a `mesh-*` name, a
|
||||
duplicate declarer, an undeclared `uses` or claim, a seat with no protocol, a scope
|
||||
mismatch, and a holder that does not answer what its seat promises. **Still to do**:
|
||||
creating a seat's streams at registration and its holder's work-queue consumer at
|
||||
assignment, which need the JetStream client wired in.
|
||||
|
||||
The refusal for an unknown claim *moved* rather than disappeared — the parser cannot judge
|
||||
it from one manifest any more, because another module may legitimately declare that seat,
|
||||
so it is registration's. The test that encoded the old rule was rewritten rather than
|
||||
deleted, and a second one pins the case the parser could not distinguish.
|
||||
|
||||
**Done**: a seat's work queue is derived and created, and a holder's worker with it. The
|
||||
JetStream client behind them is wired and verified against a running server, which also
|
||||
completes 1.4's missing half — the pure `Asserter` had no implementation until now.
|
||||
- [x] 3.10 **the ten seat renames** — done in the controller's table, the ten manifests that
|
||||
claim them, the controller's own shipped manifests, and every test. Not a migration after
|
||||
all: a holding is derived at resolution, never stored, so nothing recorded points at an old
|
||||
name (recorded as a progressive insight on ADR 0126). A **kept** rename table tells a
|
||||
manifest written against an old name what it became, because a module lives in its own
|
||||
repository and may be registered long after the catalogue stopped using one.
|
||||
|
||||
**A seat and the interface it delivers are different names.** The `git` seat became
|
||||
`mesh-git` while the `git` *provision* it delivers did not change, and the same for the
|
||||
package registry. A blanket replace got this wrong first and the failure read "the package
|
||||
registry is served on `<nil>`", which does not say "you renamed an interface" — so a test
|
||||
now pins every seat against the interface it delivers.
|
||||
|
||||
**Done when.** The fixtures are produced and consumed byte for byte by every implementation that
|
||||
claims the capability, and a module built before any of this serves its tools unchanged on the new
|
||||
@@ -196,6 +495,17 @@ module.
|
||||
**Why here.** The links exist from step 3, so the flows that are not on the bus at all can move onto
|
||||
it, and the beds that need a mesh living on NATS can finally run.
|
||||
|
||||
> **A blocker surfaced here that is not this step's to fix.** Every event name in the catalogue is
|
||||
> still written the way a routing key on the bus the mesh has is written, so the derivation design 29
|
||||
> §1 specifies turns a consumer's declaration into a subject **no emitter publishes** — thirty-seven
|
||||
> manifests, and one that cannot be composed at all. Nothing fails on the bus the mesh runs on
|
||||
> today, where a routing key is matched literally; it fails on the first mesh raised on the new bus
|
||||
> and not before, which is why wiring the controller's own subscription is what found it. Opened as
|
||||
> [issue 127](../../04-ISSUES/127-a-module-event-derives-a-subject-nothing-publishes/00-report.md).
|
||||
> It holds 4.2, 4.3 and the catch-up half of 4.5; the node-facing flows — enrolment, reports,
|
||||
> heartbeats, a build's outcome — are unaffected, because those subjects are the mesh's own and
|
||||
> derive from nothing a module declares.
|
||||
|
||||
- [ ] 4.1 **the full genesis bed** — a mesh raised on NATS from nothing and living on it: a node
|
||||
enrols over TLS with a claimed token and the enrolment user cannot read a declaration; a push
|
||||
is held while the store restarts and applies after, nothing lost or duplicated; a node that
|
||||
@@ -204,13 +514,111 @@ it, and the beds that need a mesh living on NATS can finally run.
|
||||
held by a `nak`-with-delay cycle still reaches the enrolling node, proving the reply travels
|
||||
in the payload and not the transport field the consumer's ack has claimed. The server-enforced
|
||||
permissions were proved at step 1 and are not re-proved here
|
||||
- [ ] 4.2 a build source's change reaches the builder over the bus, and the build that follows is
|
||||
the one the change asked for
|
||||
- [ ] 4.3 an installation completes over the bus, with the same outcome as the path it replaces
|
||||
- [ ] 4.4 a person's client: the account, the client that speaks the bus, and the tool surface over
|
||||
it (design 25 §7) — a module's tool invoked from another node and from a person, refused from
|
||||
an account that may not
|
||||
- [ ] 4.5 reports and catch-up: a node that was unreachable catches up rather than losing them
|
||||
— **nothing is outstanding but the bed itself.** Both links speak NATS, the composition
|
||||
happens, and every claim above has a unit test or a check against a running server behind it.
|
||||
What none of them can stand in for is a mesh raising itself, which is what this bed is — so this
|
||||
is where the code stops and the lab starts
|
||||
- [x] 4.2 a build source's change reaches the builder over the bus, and the build that follows is
|
||||
the one the change asked for — **a build is work submitted to a role now**
|
||||
([ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md)). Both sides
|
||||
are behind a seam with an implementation per bus, and on the bus being built one publish does
|
||||
what two did: the outcome is the role's own event, so the asker matches it by the id its
|
||||
request carried, the controller records it and the catalogue places it in the graph. A build
|
||||
machine needs a reply queue for nothing and a grant over nobody's inbox.
|
||||
|
||||
Checked against a running server: the round trip; a third party on the role's event hearing
|
||||
the same outcome the asker did, which is what the decision rests on; work leaving the queue
|
||||
once settled, so no second machine repeats it; work submitted with no machine holding the role
|
||||
**waiting rather than failing**, and being done when one arrives; and work a machine handed
|
||||
back coming round again.
|
||||
|
||||
The outcome carries the module name, because only the manifest says what was built and one
|
||||
message now has three readers. A failed build names none: it produced no module version, and
|
||||
the catalogue would otherwise place something that was never made.
|
||||
- [x] 4.3 an installation completes over the bus, with the same outcome as the path it replaces —
|
||||
**the installer can raise it**: a foundation template that stands up the server, writes the
|
||||
server's own settings and the mesh's first user list beside them, and starts a controller
|
||||
reaching the new bus. What remains is running it, which is 4.1's bed.
|
||||
|
||||
**The mesh composes its own user list, and at genesis there is no mesh to compose one.** So the
|
||||
installer carries the first — the controller's account at a well-known bootstrap password,
|
||||
exactly as the store is reached at `postgres:bootstrap` and the old bus at `guest:guest`, and
|
||||
rotated with them. From the controller's first composition onward the file is the controller's.
|
||||
|
||||
That surfaced a gap reading would not have found: the controller's own account exists before
|
||||
there is a controller to mint one, so nothing recorded a hash for it and its first composition
|
||||
would have left the writer out of the file it was writing — a bus nothing can connect to,
|
||||
produced by the thing connected to it. It records a hash of the credential it is using, and only
|
||||
when none is recorded, so a restart cannot put the bootstrap password back over a rotated one.
|
||||
|
||||
**The carried list and the derived one are checked against each other**, because they are two
|
||||
statements of one fact and a mesh cannot be raised twice to find out they disagreed. A template
|
||||
granting less than the controller derives produces a mesh that comes up, connects, and is
|
||||
refused on its first act, with an authorisation error naming a subject rather than the template
|
||||
that forgot it. The check earned itself at once: the composer was granting a role's whole event
|
||||
branch *and* the one event it follows, and the wider grant wins — so only the submitting half of
|
||||
a role is granted now, and what comes back is named exactly.
|
||||
- [x] 4.4 a person's client — **the account and the program are both in.**
|
||||
|
||||
**The account**: a person is not a module and holds no seat, so their authority is a list of
|
||||
tools (or `*` for an administrator) and nothing else. Held to four properties, each a way of
|
||||
being wrong that would not announce itself: nothing but tools, so a person cannot claim a
|
||||
module said something; no ack subject, because authority over a consumer that does not exist
|
||||
is authority nobody audits; no ability to answer, because a person who can answer a request is
|
||||
impersonating a module on a bus where anyone may serve a tool; and two people do not share an
|
||||
inbox. Issued, listed and revoked by command; stating what somebody may call replaces what was
|
||||
there, because a list that could only grow is a permission nobody can take back; and forgetting
|
||||
somebody takes their credential with them, or it is not a revocation.
|
||||
|
||||
**The program**: two surfaces over one thing — a command line and an MCP server — both adapters
|
||||
over the same three calls, because a second way of reaching a tool is a second thing to keep
|
||||
correct. It uses the client a module's runtime uses, so what a person may do is answered by the
|
||||
same permission list that answers it for a module and an audit has nothing separate to read.
|
||||
|
||||
Three decisions in it worth keeping. It lists what the **catalogue** has rather than what this
|
||||
credential may call: somebody seeing only their own tools cannot tell "not installed" from "not
|
||||
yours", and those need different people to fix them. A failed call says which of three things
|
||||
happened — nobody serves it, this credential may not, or the tool was slow — because the
|
||||
remedies are in three different places and without that they are one timeout and a stack trace.
|
||||
And the MCP surface decides nothing: the names are the ones a person types, the schemas are the
|
||||
modules' own, an answer is passed through unshaped, and a tool that fails comes back as a tool
|
||||
error rather than a protocol error, because the request was well-formed and the mesh answered it.
|
||||
|
||||
Both surfaces are driven against a running bus, including a host's notification being answered
|
||||
with nothing and an unknown method refused.
|
||||
|
||||
> **Design 25 §7 says "nothing is built of this before §10's bed passes", and this was built
|
||||
> before.** Recorded rather than quietly ignored: the operator asked for it, it is on the
|
||||
> critical path for nothing and blocked by nothing, and the bed it waits for is 4.1's. If the
|
||||
> bed changes what a person's client should be, this is what gets changed.
|
||||
|
||||
- [x] 4.5 reports and catch-up: a node that was unreachable catches up rather than losing them.
|
||||
|
||||
**The reports half is in and proved against a server** (3.4): held through the store's absence by
|
||||
the server rather than by the controller, superseded ones settled by the digest they carry.
|
||||
|
||||
**The catch-up half needed nothing built, and that was the answer.** It existed because a queue on
|
||||
the bus the mesh runs on today receives only what is published after it is bound, so everything
|
||||
built before the catalogue existed was announced to nobody — and on a fresh mesh that is always
|
||||
the foundation, because those are the things the catalogue needed in order to exist
|
||||
([issue 050](../../04-ISSUES/050-the-catalogue-knows-nothing-built-before-it/00-report.md)). A
|
||||
whole mechanism followed: the catalogue asks, the controller re-publishes.
|
||||
|
||||
A stream is a log and a consumer is a position in it. A consumer created later starts at the
|
||||
beginning, so the builds are simply there — asked of a running server rather than assumed, since
|
||||
the decision rested on it: three builds published with nothing listening, then a consumer created,
|
||||
and all three waiting for it. So the question of *who replays* has no answer because nothing
|
||||
replays.
|
||||
|
||||
> **This is the shape of the whole change, in one task.** Three ways to do the replay were weighed
|
||||
> — a namespace for the mesh's own voice, the controller answering a question, a consumer reading
|
||||
> from the start — and the right answer was that the bus being moved to already does it. The
|
||||
> mechanism was never about builds; it was about a queue that could not remember. **A conversion
|
||||
> that carried it across would have carried a workaround for a limitation that no longer exists**,
|
||||
> and nothing would have looked wrong.
|
||||
|
||||
Retiring it is step 5's, with the rest of what only the old bus needs: the request, the
|
||||
re-publishing, and the `replay` flag that told a consumer to register history without acting on it.
|
||||
|
||||
**Done when.** Each converted flow is proved against the behaviour it replaced, and the full genesis
|
||||
bed is green. **Observation is not in this step** — heartbeats, conditions and key-value state are
|
||||
@@ -221,16 +629,101 @@ reserves them for after the move, and a flow built ahead of its design would be
|
||||
|
||||
**Why here.** It is the only step that moves a node's bus, and it moves every node's at once.
|
||||
|
||||
- [ ] 5.1 the cutover bed: a mesh on AMQP with a predecessor stand-in on the compatibility broker
|
||||
**The order the repositories land in is part of the rollout, not paperwork.** Derived 2026-09-27
|
||||
while merging, and not obvious from any one repository, which is why it is written here rather than
|
||||
left to be re-derived under time pressure:
|
||||
|
||||
| order | repository | why it cannot be later |
|
||||
|---|---|---|
|
||||
| 1 | this one | prose; nothing deploys |
|
||||
| 2 | the sdk | comments only, and no module rebuilds for it |
|
||||
| 3 | the client library | it is what a module calls to emit, and it is where the subject is derived. Until it lands, a locally-named event is published under the local name itself |
|
||||
| 4 | the catalogue | every manifest and every module's code, renamed together. Safe only once the runtime derives |
|
||||
| 5 | the controller | **it refuses an old-style event name outright**, so landing it before the catalogue makes every unconverted module unregisterable |
|
||||
| 6 | the hosts | last, because nothing else waits on them |
|
||||
|
||||
Two properties make the sequence safe rather than merely ordered, and both are pinned by tests. A
|
||||
name already in the old form passes through the derivation untouched, so a module nobody has
|
||||
converted keeps working at every step. And a converted name derives to **exactly** the key the old
|
||||
bus published, so steps 3 and 4 change nothing on the wire — the move to the new bus is step 5.2 and
|
||||
one environment variable, not a side effect of deploying.
|
||||
|
||||
The failure this ordering avoids is issue 127's own: a publisher and a subscriber that disagree about
|
||||
a subject produce no error anywhere. Nothing logs, nothing retries, and the mesh reports itself
|
||||
healthy while reacting to nothing.
|
||||
|
||||
- [ ] 5.1 the cutover bed: a mesh on AMQP with a predecessor stand-in on the deprecated broker
|
||||
moves its bus in one rollout, every node reporting on NATS afterwards, the stand-in's own
|
||||
client still connected throughout
|
||||
- [ ] 5.2 the rollout: accounts composed, then the controller, every host and every runtime
|
||||
together; every node confirmed heard before AMQP stops
|
||||
- [ ] 5.3 the mesh's accounts removed from the compatibility broker, leaving the predecessor's users
|
||||
- [ ] 5.4 the compatibility broker retires when its condition holds — no client connected for the
|
||||
period the operator sets
|
||||
- [x] 5.2 the rollout: accounts composed, then the controller, every host and every runtime
|
||||
together; every node confirmed heard before AMQP stops.
|
||||
|
||||
**Done when.** Every node reports on NATS and the predecessor's clients never noticed.
|
||||
**Done 2026-09-28, 02:25.** Every machine reports on the new bus, the seat is held by the
|
||||
module that provides it, the old broker is unassigned and forgotten, and every credential was
|
||||
minted afresh at the end because two had been printed on the way. What it took, in the order
|
||||
it was found, each fixed on the trunk before the next step: the control plane's `serve` and
|
||||
`push` never selected the new transport (task 4.3, open until then); a machine's user was
|
||||
granted neither the asking nor the delivery of its own consumer; the account had no JetStream
|
||||
of its own; the control plane's client verified the bus's certificate by name instead of
|
||||
pinning it; the seat table's rows carried no protocol, so no role's work queue was raised; the
|
||||
build machine decided its bus from a variable its container never received; and a rotation
|
||||
put new hashes on the bus before three machines had received their new memberships — which
|
||||
is why there is now `rollout hand <node>` and a host adopts a delivered membership at start.
|
||||
The bootstrap loop — a bus that can only be raised by a declaration that can only arrive
|
||||
over that bus — was broken once, by hand: the mesh's own composed configuration started the
|
||||
server, and the controller binary was run on the node directly until the managed container
|
||||
could be rebuilt over the bus it was on.
|
||||
|
||||
- [x] 5.3 **the seat changes hands as one act.** A command takes a seat and the assignment taking it
|
||||
over, and the seat is never empty in between — the emptiness is the outage of 2026-09-27, when
|
||||
the control plane, which finds its own bus through this seat, lost the address and looped.
|
||||
**Built 2026-09-27** (`seat_holding`, migration 0039; design 26 says how it is checked), and used
|
||||
live the next night to hand `mesh-broker` from the old broker's assignment to the new one's. This
|
||||
is what 5.2 uses to move `mesh-broker` from the old
|
||||
broker's assignment to the new one's, and it is built first ([ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)).
|
||||
- [x] 5.4 **the old broker and everything that named AMQP leave the mesh** ([ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md),
|
||||
superseding [ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md)): the two modules that
|
||||
required `amqp` are removed, the broker's module is unassigned and removed (**done 2026-09-28**; the predecessor's own tooling, which rode the same adopted broker, went dark with it, as [ADR 0130](../../02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md) accepted), registration refuses
|
||||
a manifest that provides or requires `amqp`, and a whole-catalogue check asserts none does. Not
|
||||
a retirement condition — a decision, taken, with the operator's "I don't care if the predecessor
|
||||
breaks" on record ([ADR 0130](../../02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md)).
|
||||
Retiring with it: the build outcome's second announcement under the module's own name, which
|
||||
existed only so a catalogue deployed before the rename and one after both heard it.
|
||||
|
||||
> **The remote tooling goes with it too.** The predecessor's own mesh talks over that broker, so
|
||||
> shutting it down ends the path that reaches this installation's machines from a workstation.
|
||||
> The rollout is driven from the node, or before the broker stops — a sequencing constraint on
|
||||
> 5.2, not an afterthought.
|
||||
- [x] 5.5 **the AMQP transport is deleted from the control plane and the hosts**. One bus, nothing
|
||||
to select ([ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md)).
|
||||
|
||||
**Done 2026-09-28.** The control plane's old consume loop, build request, tool call, management
|
||||
API and account scoping went, and the host's old dialling and enrolment paths with them; a
|
||||
membership or token naming any other bus is refused before anything is sent. Nothing selects a
|
||||
transport any more: the variable that once did (`MESH_BUS_NATS`) now only names where the
|
||||
control plane reads its own bus credential, the way any module reads a secret. **Checked by the
|
||||
build**: neither repository's module file names the AMQP client library, so a line that still
|
||||
used it would not compile. The store-window guarantee ([issue 083](../../04-ISSUES/083-other-control-messages-are-lost-while-the-store-restarts/00-report.md))
|
||||
is tested against a bus-less fake rather than the old transport's memory, which is what let
|
||||
that memory go — the one thing it did that the stream does not (superseding a held report) is
|
||||
the staleness check on the message itself (design 25 §3).
|
||||
|
||||
Found on the way: **no build had ever recorded what it stood on.** A recipe reads its base from
|
||||
a build argument, so the digest was never in the file the builder derived edges from, and every
|
||||
order that says *bases first* — `build --on`, `build --behind`, the merge follow-up of
|
||||
[issue 131](../../04-ISSUES/131-nothing-tells-the-mesh-a-source-moved/00-report.md) — walked a
|
||||
graph with no edges. The builder now reports the bases it was handed, the control plane records
|
||||
them by artifact path, and the graph is read from the newest build of each module — a recorded
|
||||
manifest carries no `build.on`, so the edge is derived from the build or it does not exist.
|
||||
|
||||
> **The old 5.4 note is history.** It recorded that a retirement *condition* was wrong from
|
||||
> [ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) onward, which framed the old broker
|
||||
> as an ordinary provider with no end. [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md) ends that
|
||||
> framing in turn: the broker is not kept as a provider either, because AMQP is not a provision. Both
|
||||
> readings are kept here so the two reversals can be read in order.
|
||||
|
||||
**Done when.** Every node reports on NATS, and nothing of the mesh's own is left connected to the
|
||||
deprecated broker.
|
||||
|
||||
## The through-line
|
||||
|
||||
@@ -245,8 +738,10 @@ itself moves once, at the end, on one day.
|
||||
- **Observation** — research 017's, after the move, by its own design.
|
||||
- **Leaf nodes** — design 25 §11 keeps this out of scope and says so; a leaf per machine is a later
|
||||
question, noted so it is not forgotten.
|
||||
- **The predecessor's world.** It is AMQP, it cannot move, and it does not need to: its broker is
|
||||
the compatibility module until its last client is gone.
|
||||
- **The predecessor's world.** It is AMQP and it is not moving —
|
||||
[ADR 0130](../../02-DECISIONS/0130-the-predecessor-is-ending-and-its-broker-goes-with-it.md): it is
|
||||
deprecated, some of it is still running, and it is being left to stop rather than migrated. Its
|
||||
broker goes with it, unassigned like any provider whose provision nothing requires.
|
||||
|
||||
## How this list is kept true
|
||||
|
||||
|
||||
@@ -5,7 +5,10 @@ code: []
|
||||
updated: 2026-09-27
|
||||
decisions:
|
||||
- 02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md
|
||||
- 02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md
|
||||
- 02-DECISIONS/0051-shared-data-is-the-operators.md
|
||||
- 02-DECISIONS/0113-the-vault-makes-every-secret.md
|
||||
- 02-DECISIONS/0117-a-machines-uplink-is-a-seat.md
|
||||
---
|
||||
|
||||
# 29 — A node has operator accounts, and the mesh owns what lives under a home
|
||||
@@ -18,7 +21,7 @@ owned by, who a user service runs as, and — the case that surfaced this — wh
|
||||
person's `~/.ssh/config`, `~/.zshrc`, `~/.config`); the mesh, taking those over, kept the machine
|
||||
facts and dropped the human one.
|
||||
|
||||
Two things are missing, and they are one idea:
|
||||
Several things are missing, and they are one idea.
|
||||
|
||||
## 1. The account is a node fact
|
||||
|
||||
@@ -28,17 +31,14 @@ mesh already knows the node and its address, so `<account>@<node>` is then a com
|
||||
because it is exactly the fact that was silently lost — `ssh ace` failed to `ace` because nothing
|
||||
in the mesh said ace's account is `ace`.
|
||||
|
||||
It is **not** a credential. The account names a login; the key that authorises it is the
|
||||
operator's, placed as a secret or an operator-owned file, never minted by the mesh (ADR 0051).
|
||||
|
||||
## 2. A resource may live under a home, owned by its account
|
||||
|
||||
[ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) placed a
|
||||
module's *system* data — `<root>/<module>`, owned by the module. It has no analog for the other
|
||||
half of the filesystem: the things that belong under a person's home and are owned by that
|
||||
person. `~/.ssh/config`, `~/.ssh/config.d/mesh`, `~/.zshrc`, `~/.config/hal` — every one of these
|
||||
is a resource the mesh should be able to place and own, resolved against **the account's home**
|
||||
rather than a system root, and chowned to **the account** rather than to root or a module uid.
|
||||
person. `~/.ssh/config`, `~/.zshrc`, `~/.config/hal` — every one of these is a resource the mesh
|
||||
should be able to place and own, resolved against **the account's home** rather than a system
|
||||
root, and chowned to **the account** rather than to root or a module uid.
|
||||
|
||||
This is the same move as `${dir:…}`, one level over: a resource says `home: <account>` (or names
|
||||
an account requirement), and the mesh resolves the home directory and the owning uid on the node
|
||||
@@ -46,39 +46,133 @@ that account lives on. A module that writes operator config — the eventual rep
|
||||
`hal/terminal`, `hal/claude-code`, `hal/secrets` — declares its files this way and names no
|
||||
`/home/...` path, exactly as a system module now names no `/var/lib` path.
|
||||
|
||||
These are a **family**, not one module: an `ssh-client` module, a shell module, a `~/.config`
|
||||
module, each a *universal-tier* consumer of the account fact — assigned wherever a person logs in,
|
||||
which is every node, unlike the graphical stack that a capability gates.
|
||||
|
||||
## 3. The whole of `~/.ssh` is the mesh's — with one boundary drawn inside it
|
||||
|
||||
The predecessor owned a single file (`~/.ssh/config`) and left the rest alone; it drifted, because
|
||||
owning one file beside foreign ones is not owning anything. The mesh should own **the directory**:
|
||||
create `~/.ssh` at `0700`, chown it to the account, and own the files it places there —
|
||||
|
||||
- **`config`** (or the mesh's region of it): the `Host` blocks for every other node, composed
|
||||
from the roster;
|
||||
- **`known_hosts`**: authoritative, so the "Host key verification failed / accept-new" dance that
|
||||
cost real time during enrolment simply ends;
|
||||
- **`authorized_keys`**: who may log into this account, governed centrally rather than by whichever
|
||||
key happened to be pasted where.
|
||||
|
||||
**The boundary — and it is the reason this is safe:** `~/.ssh` is the one directory where a wrong
|
||||
declaration locks a person out of their own machine. So the mesh's *found-vs-owned* semantics
|
||||
([ADR 0126](../../02-DECISIONS/0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md),
|
||||
adoption) apply *inside* the home directory. The mesh **owns** the directory and the files above; it
|
||||
**holds as found — never rewrites, never removes** — the operator's own contents: their **private
|
||||
keys** and their **personal drop-ins** (`config.d/personal`, the personal `Host` aliases a
|
||||
workstation carries, exactly as `hosts.local` is the home the mesh never rewrites for `/etc/hosts`).
|
||||
Reconcile removing an unassigned `config.d/mesh` is fine; the same logic aimed at `id_ed25519` or an
|
||||
operator's own `authorized_keys` entry is a lockout. This is the login-channel cousin of the rule
|
||||
[ADR 0125](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md) draws for the uplink and the sshd
|
||||
module draws for the firewall: **the mesh must never be able to arrange the one failure that severs
|
||||
its own way back in.** The carve-out is not a convenience; it is that rule, in `~/.ssh`.
|
||||
|
||||
## 4. Keys are the mesh's to generate — through a CA, and existing keys are adopted, not replaced
|
||||
|
||||
Key *generation* is the mesh's, not each node's improvising its own. The clean form is an **SSH
|
||||
certificate authority as a seat**, the sibling of the TLS internal CA the mesh already runs:
|
||||
|
||||
- **Host certs.** The mesh signs each node's host key. Every node's `known_hosts` becomes one line
|
||||
— `@cert-authority *.<suffix> <mesh-CA-key>` — and nothing is distributed per node; a new node is
|
||||
trusted the instant its host key is signed.
|
||||
- **User certs.** The mesh signs a cert naming the principals (accounts) allowed. Every node's
|
||||
`authorized_keys` / sshd `TrustedUserCAKeys` becomes one trust line — no N×N key spraying — and
|
||||
short-lived certs give rotation for free
|
||||
([ADR 0114](../../02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md)).
|
||||
- The **CA private key is the mesh's**, a secret the vault makes
|
||||
([ADR 0113](../../02-DECISIONS/0113-the-vault-makes-every-secret.md)).
|
||||
|
||||
**Three kinds of key, and only one is never minted.** Host keys (server identity) and pure
|
||||
machine-to-machine keys the mesh may generate end to end. The operator's **personal** private key —
|
||||
possibly on a hardware token, possibly used from an off-mesh laptop — the mesh **signs into a cert
|
||||
but never generates**; that, and only that, is the residue of
|
||||
[ADR 0051](../../02-DECISIONS/0051-shared-data-is-the-operators.md). So "keys are mesh-owned" and
|
||||
"the operator's login key is the operator's" reconcile: the mesh owns the CA and the signing; it
|
||||
holds the human's private half, never mints it.
|
||||
|
||||
**Existing keys are not lost.** Taking ownership is *adoption*, not regeneration: a key already on a
|
||||
machine is recorded and signed, not overwritten. The mesh gains authority over `~/.ssh` — it does
|
||||
not clear it. An enrolling node's host key and the operator's existing key are carried forward; the
|
||||
found-vs-owned boundary of §3 is exactly what guarantees nothing already there is destroyed.
|
||||
|
||||
## 5. How it is distributed: the controller composes, the node applies
|
||||
|
||||
None of this needs a node to discover the mesh, and none of it needs a control-plane module of its
|
||||
own. The ssh files are **roster facts**
|
||||
([ADR 0128](../../02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md)): once the
|
||||
roster view carries a node's **host key** and its **account** beside its name and address, the
|
||||
`ssh-client` module ships a template for `known_hosts`, `config` and `authorized_keys`, and the
|
||||
controller renders each node's copy from the full roster and pushes it. The mesh owns the data; the
|
||||
module owns ssh's format; the control plane gains no ssh syntax. It is the same act as composing a
|
||||
peer list or `/etc/hosts` — which is why there is **no novox-only "mesh-ssh" module**: the
|
||||
centralization is the controller's composition, not a module that runs somewhere. Only non-secret
|
||||
facts travel (names, addresses, accounts, host keys, the CA public key); the private key stays the
|
||||
operator's, placed as an operator-owned file, referenced by path.
|
||||
|
||||
## 6. The two modules, and the seat between them
|
||||
|
||||
- **`sshd`** (server, every node) — manages sshd, owns and **reports** its host key so the roster
|
||||
carries it, and trusts the user CA.
|
||||
- **`ssh-client`** (client, every node) — owns `~/.ssh` per §3, consumes the roster and the CA
|
||||
public key.
|
||||
- **`the-ssh-ca`** (a seat, held on the control node) — signs host and user certs.
|
||||
|
||||
They meet at the account and the CA, not at a bespoke module. The `sshd` server side already exists;
|
||||
the client/identity side and the CA are the open pieces.
|
||||
|
||||
## Why now, and why not yet
|
||||
|
||||
**Why it matters:** when HAL retires, the generators that keep `~/.ssh/config`, shell config and
|
||||
the operator's `~/.config/hal` current retire with it. Without this, adding a node stops adding
|
||||
its ssh alias, and a fresh machine has no operator dotfiles at all — the mesh would run every
|
||||
service and leave the human unable to work on the box. The account is also load-bearing for
|
||||
correctness already: `ssh <node>` (issue 122's cousin), user-scoped systemd units, and any file a
|
||||
person rather than a daemon must own.
|
||||
**Why it matters:** when HAL retires, the generators that keep `~/.ssh`, shell config and the
|
||||
operator's `~/.config/hal` current retire with it. Without this, adding a node stops adding its ssh
|
||||
alias and its trust, and a fresh machine has no operator dotfiles at all — the mesh would run every
|
||||
service and leave the human unable to work on the box.
|
||||
|
||||
**Why not build it reflexively:** it is a real addition to the node model and the resource model,
|
||||
and it must be gotten right, not smuggled in beside a firewall fix. Open questions to settle
|
||||
first:
|
||||
**Why not build it reflexively:** it is a real addition to the node model, the resource model, and
|
||||
the seat set, and must be gotten right. The mechanism half is now settled — ADR 0128 is what lets
|
||||
the ssh files be templates with no control-plane format — so what remains to decide here is the
|
||||
model:
|
||||
|
||||
- **One account or several per node?** A workstation has one human; a shared box might have more.
|
||||
The model should allow more than one without forcing the common case to name it.
|
||||
- **Where the login key lives.** An operator-owned file (ADR 0051) or an accepted secret — never
|
||||
minted. The account fact and the key that authorises it are separate, and only the first is the
|
||||
mesh's to generate.
|
||||
- **The boundary with `sshd`.** The `sshd` module (server side) already exists. This is the
|
||||
*client* and *identity* side: the account a node offers, and the home-scoped files an operator
|
||||
needs. They meet at the account but are not the same module.
|
||||
- **Multi-operator.** Today there is one human. The model should not assume it, but the first
|
||||
cut may serve one and leave the shape open.
|
||||
Allow more than one without forcing the common case to name it.
|
||||
- **The CA's shape.** Host-cert and user-cert principals, cert lifetime and renewal, where the CA
|
||||
runs (a seat on the control node). The one thing fixed: the operator's personal key is signed,
|
||||
never minted.
|
||||
- **Adoption of existing keys.** How an enrolling node's host key and an operator's existing key are
|
||||
recorded and signed rather than replaced — the found-vs-owned boundary, made concrete for keys.
|
||||
- **The `sshd` boundary.** Server side exists; this is the client, the identity, and the CA.
|
||||
- **The ssh-agent.** An agent is a *user-scoped service running as the account* — the first concrete
|
||||
case of the user services §2 anticipates. It holds the operator's private key in memory; the mesh
|
||||
declares the unit and sets `AddKeysToAgent`/`IdentityAgent` in `config`, and still never sees the
|
||||
private half. Agent *forwarding* wants a policy, not a default: with user certs it is largely
|
||||
unnecessary, and forwarding an agent into a node exposes the operator's keys to that node's root —
|
||||
so prefer certificates and `ProxyJump` over forwarding.
|
||||
|
||||
**Not urgent, not blocking.** ssh and dotfiles work today because HAL's generators still run as
|
||||
the substrate. This becomes load-bearing in the node-by-node retirement phase, not before — which
|
||||
is the right time to build it, once the account model is decided here.
|
||||
**Not urgent, not blocking.** ssh and dotfiles work today because HAL's generators still run as the
|
||||
substrate. This becomes load-bearing in the node-by-node retirement phase, not before — which is the
|
||||
right time to build it, once the account and CA model are decided here.
|
||||
|
||||
## References
|
||||
|
||||
- The gap was found generating `~/.ssh/config` from the *HAL* registry (`hal/terminal`'s
|
||||
postConfigure hook), which the nox mesh has no equivalent for.
|
||||
- [ADR 0128](../../02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md) — the roster
|
||||
fact mechanism that renders the ssh files, format owned by the module.
|
||||
- [ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) — the
|
||||
system-path placement this mirrors for home paths.
|
||||
- [ADR 0051](../../02-DECISIONS/0051-shared-data-is-the-operators.md) — why the login key stays
|
||||
the operator's, never minted.
|
||||
- [ADR 0051](../../02-DECISIONS/0051-shared-data-is-the-operators.md) — why the operator's personal
|
||||
key is signed, never minted.
|
||||
- [ADR 0113](../../02-DECISIONS/0113-the-vault-makes-every-secret.md) — the CA key is a secret the
|
||||
vault makes; [ADR 0114](../../02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md)
|
||||
— short-lived certs as rotation.
|
||||
- [ADR 0125](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md),
|
||||
[ADR 0126](../../02-DECISIONS/0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md) —
|
||||
the never-sever-the-channel rule and the found-vs-owned semantics, applied here to `~/.ssh`.
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: proposed
|
||||
code: []
|
||||
updated: 2026-09-27
|
||||
decisions:
|
||||
- 02-DECISIONS/0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md
|
||||
- 02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md
|
||||
---
|
||||
|
||||
# 30 — The mesh updates itself on a push
|
||||
|
||||
**Today the mesh does not update itself; a person drives the pipeline by hand, and one class of
|
||||
change freezes it.** A code change lands in `mesh-controller` or `mesh-catalog`, and getting it onto
|
||||
the machines is a sequence somebody types. The predecessor's pipelines rebuilt and redeployed on a
|
||||
push without anyone watching; the successor should too. This records the process as it is done by
|
||||
hand now — so it can be read, and then coded — and the two things that make it more than "add a
|
||||
webhook".
|
||||
|
||||
## The process, as done by hand
|
||||
|
||||
**An ordinary (non-breaking) change** — new module code, a bug fix, a manifest tweak that changes no
|
||||
seat or schema:
|
||||
|
||||
1. `module moved <module> <commit>` — tell the mesh its source advanced (the controller repo has no
|
||||
trigger, so this is manual; the catalogue's webhook does it automatically — see below).
|
||||
2. `build --behind` (or `build <repo> [--ref] [--path <subdir>]`) — the build machine rebuilds and
|
||||
records the new image.
|
||||
3. The mesh **reconciles on its own**: the module's declaration now names the new image, the next
|
||||
push/heartbeat sends it, and the host swaps the container. For the control plane this is a
|
||||
self-upgrade — the running controller composes its own new image and the host replaces it. No
|
||||
restart is typed.
|
||||
|
||||
**A breaking change** — a manifest schema the controller parses differently (a fact's shape, a
|
||||
seat's name), where the new control plane cannot read the manifests the old one stored:
|
||||
|
||||
4. Land the code (controller + catalogue together — they are one change).
|
||||
5. Rebuild + deploy the new controller (steps 1–3). **The moment it is live it refuses the
|
||||
still-old-shape stored manifests, and composition freezes for every node that runs an affected
|
||||
module.** Running services are untouched; only new declarations stop.
|
||||
6. **Re-register each affected manifest under the new shape**, which the *new* controller accepts —
|
||||
`module add <file> -source <repo> -ref <ref> -commit <commit>`. This writes the manifest to the
|
||||
store without a build, so it is the fast way to lift the freeze. (The controller container is
|
||||
distroless: `docker cp` the file to the container root `/x.json`; `/tmp` does not exist; the
|
||||
root filesystem is writable. The file is lost when the container is recreated on the next image
|
||||
swap, so copy it *after* the swap.)
|
||||
7. `push --behind`, then verify `status` is clean and `seats` (or the relevant surface) shows the
|
||||
new shape held by the right holders.
|
||||
|
||||
The freeze in a breaking change has been paid three times in one session (a fact-shape change, the
|
||||
`/etc/hosts` region, a seat rename); each time it lasted seconds and no service dropped. It is
|
||||
recoverable, but it is not something a push should trigger unwatched — which is the crux of what
|
||||
automating this must solve.
|
||||
|
||||
## Why it is more than "add a webhook"
|
||||
|
||||
### 1. The trigger today is HAL's, not the mesh's
|
||||
|
||||
Build-on-push works for the catalogue because its repository has a Gitea webhook pointing at
|
||||
`http://host.docker.internal:9877/webhook/gitea` — and **that receiver is `hal-gitea-tools.service`**
|
||||
(`~/.hal/modules/hal/gitea/tools/server.js`), a *predecessor* component. The nox builder consumes
|
||||
build work; it does not receive Git events. So the mesh's own build pipeline currently rides on a
|
||||
HAL service, and:
|
||||
|
||||
- the `mesh-controller` repository was never wired to it, which is why the control plane is the one
|
||||
thing that does **not** self-update — every controller deploy this session was `module moved` +
|
||||
`build` by hand;
|
||||
- when HAL is retired, build-on-push stops for the whole mesh.
|
||||
|
||||
**The mesh needs its own forge-webhook→build trigger**, a nox component (a module, and likely a
|
||||
seat — `mesh-forge-trigger` or folded into the git seat's holder) that receives Git events and turns
|
||||
them into build work over the broker, for **every** repository including `mesh-controller`. Replacing
|
||||
`hal-gitea-tools` is the concrete first build. Its logic already exists to copy: match the pushed
|
||||
repository (and changed paths, for a monorepo like the catalogue) against the build-context
|
||||
repository of every registered module, and rebuild the matches.
|
||||
|
||||
### 2. The builder validates too — and a breaking change deadlocks it
|
||||
|
||||
The build machine embeds the same catalogue package the controller does, so **it validates a
|
||||
manifest against its own compiled-in seat/schema set**. A breaking change therefore couples *four*
|
||||
things, not two: the controller, the **builder**, every affected manifest, and every node's host.
|
||||
This session's seat rename rebuilt the controller but not the builder, and the stale builder then
|
||||
refused every manifest claiming a renamed seat.
|
||||
|
||||
Worse, one rename **deadlocked** the builder: the build machine's own seat was renamed
|
||||
(`the-build-machine` → `mesh-build-machine`). To refresh the builder you must build it; to build it
|
||||
the *running* (old) builder must accept the new builder's manifest — which claims the new name it
|
||||
does not know. The old builder cannot build the new builder. Escapes:
|
||||
|
||||
- **Never rename a seat whose holder validates manifests** in an ordinary pass — the build machine's
|
||||
seat belongs with the deferred delivering seats (ADR 0121). Reverting `mesh-build-machine` to
|
||||
`the-build-machine` (deferred) lets the old builder build the new builder, which then knows the
|
||||
new names.
|
||||
- Or bootstrap a new builder image **out of band** (build locally, publish to the registry, register
|
||||
the module at that digest), the way genesis loads the first builder — bypassing the old builder's
|
||||
validation once.
|
||||
|
||||
Either way, self-update for breaking changes needs a **transition discipline** so a push does not
|
||||
auto-freeze: the new control plane (and builder) should accept the *old and new* shape together for
|
||||
one release — deprecated aliases in the seat set, a schema that reads both — then a later release
|
||||
drops the old. With that, a breaking change rolls out on a push like any other: everything reads
|
||||
both, the manifests migrate, the compatibility is removed. Without it, self-update would simply
|
||||
automate the freeze.
|
||||
|
||||
## What to build
|
||||
|
||||
- **A nox forge-webhook trigger** (replaces `hal-gitea-tools`): receives Git events for every mesh
|
||||
repository, dispatches build work to the builder over the broker, and records `module moved`
|
||||
automatically. Wire `mesh-controller` to it so the control plane self-updates like everything else.
|
||||
- **A transition discipline for breaking changes**: the control plane and builder accept old+new for
|
||||
one release; the tooling that lands a schema/seat change emits the compatibility shim and the
|
||||
follow-up that removes it. This is what makes step 4–7 above safe to trigger unwatched.
|
||||
- **Config/package modules need no builder** — `module add` registers their manifest directly
|
||||
(this is how the uplink managers and the re-registrations above were done). Only image-bearing
|
||||
modules need the build machine, which narrows what the deadlock above can block.
|
||||
|
||||
## Why now, and why not yet
|
||||
|
||||
**Why it matters:** self-update is the difference between a mesh a person maintains by typing
|
||||
pipeline steps and one that maintains itself, and it is a stated goal (parity with the predecessor's
|
||||
pipelines). The HAL trigger dependency also makes it a retirement blocker: build-on-push dies with
|
||||
HAL.
|
||||
|
||||
**Why not reflexively:** the trigger is a new component with the broker and forge in its blast
|
||||
radius, and the transition discipline changes how every breaking change is written. Both should be
|
||||
designed, not bolted on beside a freeze. The manual process above is the interim, and it works.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0121](../../02-DECISIONS/0121-a-system-seat-is-named-for-its-scope-and-modules-define-their-own.md)
|
||||
— the seat rename whose migration and builder deadlock this record is drawn from
|
||||
- [ADR 0128](../../02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md) — the
|
||||
fact-shape change that first showed the breaking-change freeze
|
||||
- `hal-gitea-tools.service` (`~/.hal/modules/hal/gitea/tools/server.js`) — the predecessor webhook
|
||||
receiver on `:9877` the mesh currently rides on
|
||||
- mesh-controller `cmd/mesh-builder` (the build machine), `internal/catalogue` (the seat/schema
|
||||
validation the builder shares with the controller)
|
||||
@@ -0,0 +1,65 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: proposed
|
||||
code: []
|
||||
updated: 2026-09-27
|
||||
decisions:
|
||||
- 02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md
|
||||
---
|
||||
|
||||
# 31 — A module declares its fail2ban jail, and the mesh composes them per node
|
||||
|
||||
**A node's intrusion filter should be composed from the modules it runs, the same way its firewall
|
||||
is.** The mesh already derives a node's nftables ruleset from every assigned module's `listens` and
|
||||
`guards` (the `Filtering` mechanism). fail2ban is the same shape and is not modelled: a module that
|
||||
runs an authenticating service — postgres, mssql, mailu — has a jail (a filter that reads its log
|
||||
and a jail stanza that bans on it), and which jails a node's fail2ban runs should be exactly the
|
||||
jails of the modules assigned to that node.
|
||||
|
||||
The predecessor did this with per-module files: `postgres` shipped `postgres-auth.conf`, `mssql`
|
||||
shipped `mssql-auth.conf`, `mailu` shipped `mailu.conf`, and the node's fail2ban read whichever were
|
||||
present. When HAL retired on novox those became dangling symlinks — fail2ban ran the jails only from
|
||||
memory, and a restart would have dropped them. The base was salvaged (the fail2ban module now ships
|
||||
`sshd`, `recidive`, and the `ignoreip` that spares the mesh's own range), but the **service jails
|
||||
are gone**, because no nox module declares one yet.
|
||||
|
||||
## The shape
|
||||
|
||||
- **A module declares its jail in its manifest**, naming no node and no path (ADR 0112): the filter
|
||||
(the failregex, or a stock filter it uses) and the jail stanza (port, logpath, maxretry, bantime).
|
||||
The `postgres` module says what a postgres brute-force looks like and how to ban it; it does not
|
||||
say on which machine, because it does not know.
|
||||
- **The mesh composes them per node.** For each node, the jails of its assigned modules are gathered
|
||||
and written into the fail2ban holder's `jail.d/` (and filters into `filter.d/`), exactly as
|
||||
`listens`/`guards` are gathered into the node's firewall. So a node running postgres gets the
|
||||
postgres jail; a node not running it does not. The `node-intrusion-prevention` holder receives
|
||||
them the way a provider receives its consumers' contributions.
|
||||
- **The base stays the fail2ban module's**: `sshd`, `recidive`, and the `ignoreip` naming
|
||||
`${machine:mesh-range}` so a tunnel peer is never banned.
|
||||
|
||||
## Why this, and not the module writing the file itself
|
||||
|
||||
A module could declare a `file` resource at `/etc/fail2ban/jail.d/<x>.conf` directly. Rejected: the
|
||||
path is the fail2ban holder's to own (one module owns `jail.d`, as one module owns the firewall
|
||||
table), the jail's logpath and defaults want the mesh's composition (the `ignoreip`, the ban action
|
||||
the node uses), and two modules writing into one directory is the collision the seat/holder model
|
||||
exists to prevent. The module declares *what its jail is*; the holder's composition decides *how it
|
||||
lands* — the same split as `listens` (the module says the port; the mesh says the rule).
|
||||
|
||||
## Why now
|
||||
|
||||
fail2ban on novox currently runs the service jails from memory only; the next restart drops them
|
||||
(the `ignoreip` is safe on disk, so the mesh-partition risk is closed, but postgres/mssql/mailu
|
||||
auth-banning would be lost). This is the mechanism that restores them properly, and it is needed as
|
||||
each of those modules migrates to the other nodes — ace running postgres should get the postgres
|
||||
jail, composed from the postgres module's manifest, without anyone editing a node.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) — a module
|
||||
names no node or path; its jail is declared the same way its `listens` are
|
||||
- mesh-controller `internal/catalogue/adoption.go` (`Filtering` — the firewall composition this
|
||||
mirrors), `internal/catalogue/manifest.go` (`Listens`/`Guards`, the fields a jail field sits
|
||||
beside)
|
||||
- mesh-catalog `modules/fail2ban` (the base: sshd, recidive, ignoreip); the service modules
|
||||
(`postgres`, `mssql`, `mailu`) that will declare jails
|
||||
@@ -0,0 +1,502 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: in-progress
|
||||
code:
|
||||
- mesh-controller internal/catalogue/declaration.go
|
||||
- mesh-controller internal/catalogue/manifest.go
|
||||
- mesh-controller internal/link/serve.go
|
||||
- mesh-controller internal/link/bus.go
|
||||
- mesh-controller internal/broker/nats.go
|
||||
- mesh-controller internal/inventory/nodes.go
|
||||
- mesh-host internal/apply/apply.go
|
||||
- mesh-tools src/main.ts
|
||||
- mesh-catalog modules/mesh-catalog
|
||||
updated: 2026-09-28
|
||||
decisions:
|
||||
- 02-DECISIONS/0126-a-module-declares-its-own-seats.md
|
||||
- 02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md
|
||||
- 02-DECISIONS/0128-the-mesh-bus-is-required-not-ambient.md
|
||||
- 02-DECISIONS/0106-the-bus-is-nats.md
|
||||
- 02-DECISIONS/0041-events-are-a-relationship.md
|
||||
- 02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md
|
||||
- 02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md
|
||||
- 02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md
|
||||
- 02-DECISIONS/0135-a-module-version-prepares-its-state-before-it-runs.md
|
||||
- 02-DECISIONS/0136-a-step-gates-its-module-not-the-machine.md
|
||||
- 02-DECISIONS/0134-the-mesh-says-what-it-applied.md
|
||||
---
|
||||
|
||||
# 32. What a module declares, and what the bus makes of it
|
||||
|
||||
**A module that speaks to the mesh requires the bus, and receives what it needs to connect**
|
||||
([ADR 0128](../../02-DECISIONS/0128-the-mesh-bus-is-required-not-ambient.md)). What a module
|
||||
declares are *relationships*; subjects, streams, consumers and permissions are all derived from
|
||||
those, and a manifest never contains one.
|
||||
|
||||
> **Revised 2026-09-26.** This document opened by calling the bus *ambient* — "no module requires
|
||||
> it, the way no module requires a filesystem". Two counts say otherwise: of 72 modules in the
|
||||
> catalogue, **49 take a broker credential and 23 do not**, so an ambient connection would mint an
|
||||
> account for a third of the catalogue that never speaks; and the 49 each hand-write the path it
|
||||
> lands at, which is provisioning done badly by hand. The bus is required, and a module that does
|
||||
> not require it has no account at all.
|
||||
|
||||
**The requirement delivers the connection; the declarations shape the authority.** `requires:
|
||||
mesh-bus` says *this module talks to the mesh* and grants no subject by itself. `emits`,
|
||||
`consumes`, `tools`, `uses` and a declared seat say what it may say and hear. Declaring a subject
|
||||
without requiring the bus is incoherent and refused at registration.
|
||||
|
||||
This document is the declaration model. [Design 25](25-the-bus-on-nats.md) is the bus itself —
|
||||
subjects, streams, accounts, enrolment — and stays the authority on the wire.
|
||||
[Design 19](19-the-module-protocol.md) is the specification an SDK implements, and is rewritten
|
||||
onto this in step 3 of [ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md).
|
||||
|
||||
## 1. A module names locally; the mesh derives the subject
|
||||
|
||||
This is the load-bearing rule.
|
||||
[ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) says a module
|
||||
definition names no node, mesh or path. A transport address is the same class of thing: if
|
||||
manifests held literal subjects, reorganising the subject space would mean editing every module in
|
||||
the catalogue, and the mesh would have hundreds of copies of a decision it made once.
|
||||
|
||||
| declared | derived |
|
||||
|---|---|
|
||||
| `emits: order.placed` | publish on `mesh.mod.<module>.event.order.placed` |
|
||||
| `consumes: billing.order.placed` | durable consumer on `mesh.mod.billing.event.order.placed` |
|
||||
| `tools: status` | queue-group subscription on `mesh.mod.<module>.tool.status` |
|
||||
| seat `telegram-sender`, `accepts: send` | work-queue consumer on `mesh.seat.telegram-sender.accept.send` |
|
||||
| `uses: telegram-sender` | publish on that seat's `accept` subjects, and nothing else |
|
||||
|
||||
**Wildcards, and they are the mesh's rather than a bus's.** *Added 2026-09-27, from
|
||||
[issue 127](../../04-ISSUES/127-a-module-event-derives-a-subject-nothing-publishes/00-report.md).* A
|
||||
consumer may write `*` for one name and `**` for the rest: `*.download.completed` is that event from
|
||||
any module, and `**` on its own is every event in the mesh, which an audit logger wants and says in
|
||||
one token. Spelled this way rather than the wire's, for the reason everything else here is local —
|
||||
the bus the mesh runs on today spells these `*` and `#`, the one being built spells them `*` and
|
||||
`>`, and a manifest naming either would stop being true when the wire changed. An emitted event
|
||||
carries no wildcard: it names one event.
|
||||
|
||||
**A module publishes under its own name, and an event about a role belongs to the seat.** *Added
|
||||
2026-09-27, same source.* The bus enforces that a namespace belongs to the module it is named for, so
|
||||
an event named for somebody else cannot be published at all. Where the event is really about a role —
|
||||
"the artifact store accepted an image" — the seat is the right home, because that name outlives
|
||||
whoever fills it, and a consumer written against the holder's own name breaks when the holder
|
||||
changes. **Not yet possible in practice**: seats carry protocol in the manifest and in the permission
|
||||
model, and the shared library has no way for a module to publish on one. Until it does, such an event
|
||||
lives under the emitting module's own name and the consumer carries that coupling.
|
||||
|
||||
**How the rule is checked, because it was not.** *Added 2026-09-27, same source.* Two checks, because
|
||||
the mistake happens at two scales. Per manifest, at registration: an event is a local name, and the
|
||||
old bus's form is refused with the name to write instead. Across the whole catalogue, as a test:
|
||||
where a consumed event's emitter is present, it must emit that event. The second cannot demand a live
|
||||
emitter for everything — a module lives in its own repository and may be installed long before the
|
||||
one whose events it wants — so it says nothing about an absent emitter and everything about a present
|
||||
one. **A subscription that matches nothing is not an error, it is silence**, which is why nothing
|
||||
reported thirty-seven manifests being wrong the same way.
|
||||
|
||||
**It is `tools:`, not `serves:`.** Revision, found while implementing: the manifest already uses
|
||||
`serves` for the facts a consumer needs in order to reach a provision, and two meanings under one
|
||||
key in the file a module author reads most is a footgun. Worth noting that until now a module's
|
||||
tools were not declared at all — they were known only at runtime, from an environment variable in
|
||||
its image — so declaring them is new, and is what lets the mesh check that a module claiming a
|
||||
seat answers what that seat's protocol promises.
|
||||
|
||||
**The `event` / `tool` / `accept` token is load-bearing, not decoration.** Revision, found while
|
||||
defining the streams: a stream is defined by a subject filter, so a namespace holding both a
|
||||
module's events and its tool calls cannot be filtered into an events stream without capturing
|
||||
every tool invocation in the mesh — and a tool call must never be persisted
|
||||
([design 25](25-the-bus-on-nats.md) §3 keeps tools on core NATS, where a lost call is a timeout the
|
||||
caller already handles). The kind token is what makes `mesh.mod.*.event.>` a safe filter. The
|
||||
first draft of this table had no token, which reads better and cannot be implemented.
|
||||
|
||||
**The test this must pass: the manifest survives the wire changing.** Reorganise the subject space
|
||||
and every manifest in the catalogue is still correct. That is the property
|
||||
[ADR 0039](../../02-DECISIONS/0039-what-the-sdk-holds-and-refuses.md) gave the sdk, applied to
|
||||
declarations.
|
||||
|
||||
**A handler names an event the way its manifest does.** *Added 2026-09-27, found while fixing issue
|
||||
127.* The runtime handed a handler the event name alone, so a manifest declaring
|
||||
`consumes: builder.built` produced a pattern that could never match the key it was compared against,
|
||||
and a module consuming one event from two emitters could tell them apart only by reading a header.
|
||||
The subject already carries the emitter, so the key a module sees names it too — which makes a
|
||||
disagreement between a manifest and the code a typo rather than a category error.
|
||||
|
||||
## 2. Three namespaces, and nothing else
|
||||
|
||||
**Its own** — `mesh.mod.<module>.>`. Its events and its tools. Nothing else may publish into it,
|
||||
so an event's source is a fact the bus enforces rather than a claim in the body.
|
||||
|
||||
**Seats it holds** — `mesh.seat.<seat>.>`. Full participation: consume what the seat accepts,
|
||||
publish what it emits, serve what it serves.
|
||||
|
||||
**Seats it uses** — publish only, and only on the `accepts` half. A sender cannot subscribe to a
|
||||
seat's inbound subject and watch other modules' traffic, and cannot publish the seat's outbound
|
||||
events and lie about outcomes.
|
||||
|
||||
A module naming anything outside these three is refused at registration. The whole permission set
|
||||
is derivable from the declaration; nobody writes an access rule.
|
||||
|
||||
## 3. Queues are derived, never declared
|
||||
|
||||
A module says what it reacts to, not how delivery works. Each `consumes` becomes one durable
|
||||
consumer; a seat's `accepts` becomes one work-queue consumer with a queue group named for the
|
||||
seat. The module does not name them, does not know their names, and cannot misconfigure them —
|
||||
and the controller stays the only writer of stream and consumer definitions
|
||||
([design 25](25-the-bus-on-nats.md) §3).
|
||||
|
||||
**The mesh's own seats carry protocol too.** *Added 2026-09-27,
|
||||
[ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md).* A seat declared by a
|
||||
module says what it accepts, emits and serves; the `mesh-*` set said only who does a job. So the mesh
|
||||
had roles it could not describe — a build machine with three audiences for one outcome and no way to
|
||||
derive a grant for any of them, and an event genuinely about a role with nowhere to live but the
|
||||
namespace of whichever module happens to hold it. The mesh's seats now take the same three fields, and
|
||||
the same machinery derives the holder's authority, its work queue and its consumers.
|
||||
|
||||
**So a build is work submitted to a role, like any other.** The build machine seat accepts a build and
|
||||
emits an outcome, and the dedicated branch that carried builds retires: a work queue shared by several
|
||||
machines is exactly what `accepts` already is, and a second mechanism for it is two places a permission
|
||||
can be wrong.
|
||||
|
||||
**And one publish reaches three audiences without anybody's inbox being opened.** A build's outcome is
|
||||
the seat's own event: whoever asked matches it by the id their request carried, the controller records
|
||||
it, the catalogue places it in the graph. That is the fan-out a shared exchange gave for free, written
|
||||
as a subject the mesh derived instead of a topology somebody configured — and it is why a holder needs
|
||||
no permission to publish into an asker's inbox, which is the one grant design 25 §4 refuses by name.
|
||||
|
||||
**Retention belongs to whoever owns the namespace, not to a consumer.** A seat declares how long
|
||||
its inbound backlog survives, because that is a property of the service:
|
||||
|
||||
```
|
||||
seat: telegram-sender
|
||||
scope: mesh
|
||||
accepts: send retain 7d
|
||||
emits: delivered, failed
|
||||
serves: status
|
||||
```
|
||||
|
||||
If each consumer could tune it, the mesh's durability would be an emergent property of whichever
|
||||
manifest was edited last.
|
||||
|
||||
**Why a module's own events do not carry their own retention, though the same rule would allow
|
||||
it.** A seat owns its namespace and gets a stream of its own, so it can say. A module's events
|
||||
share one `EVENTS` stream, and three facts about JetStream decide that they must:
|
||||
|
||||
- **Storage is not a property of a subject.** A subject is only an address; a *stream* is a
|
||||
separate object that captures subjects matching a filter. So "this topic is durable" is always
|
||||
really "some stream covers it", and something has to create that stream.
|
||||
- **Overlapping streams are refused, not merged.** Verified against the server: a per-module
|
||||
stream beside a shared `mesh.mod.*.event.>` is rejected with *subjects overlap with an existing
|
||||
stream*. So "one stream by default, its own for a module that wants different retention" is not
|
||||
available — it is all of one or all of the other, and a filter cannot express an exception
|
||||
either.
|
||||
- **A stream per module breaks cross-module consumption.** An audit logger consuming every
|
||||
module's events is one consumer on one stream today; with a stream each it becomes one consumer
|
||||
per module, created and destroyed as modules come and go.
|
||||
|
||||
So: one stream, and **per-subject caps** for the fairness that actually matters — a noisy emitter
|
||||
cannot evict a quiet one, which is verified (a cap of three, ten messages on one subject and one
|
||||
on another, leaves four). What is genuinely unavailable is a different *age* per module, because
|
||||
JetStream ages per stream and not per subject. A module that truly needs its own retention has a
|
||||
way to say so: declare a seat, which owns its namespace and gets its own stream.
|
||||
|
||||
**Scope gives per-node workers without a new concept.** A module running on three nodes that each
|
||||
need their own queue declares a node-scoped seat: one holder per node, three queues, same
|
||||
machinery. Mesh-scoped and node-scoped seats already exist; here they do the work of "one shared
|
||||
service" versus "one worker per machine".
|
||||
|
||||
## 4. Five relationships
|
||||
|
||||
| | provision | event | job | state | tool |
|
||||
|---|---|---|---|---|---|
|
||||
| shape | 1:1 resource | 1:many | N:1 | 1:1 | 1:1 |
|
||||
| addressed to | a provider | the emitter's own namespace | a **seat** | one node | a module or seat |
|
||||
| who must act | the provider | nobody | exactly one holder | that node | the server |
|
||||
| credential | sealed, per consumer | none | none | none | none |
|
||||
| reply | — | none | none, or an event later | a report | awaited |
|
||||
| retention | — | age and size | work queue, explicit ack | **last per subject** | none |
|
||||
| declared | `provides`/`requires` | `emits`/`consumes` | seat `accepts` / `uses` | the mesh's own | `serves` |
|
||||
|
||||
**Job** is the one [ADR 0041](../../02-DECISIONS/0041-events-are-a-relationship.md) had no room
|
||||
for. Its table has events at 1:many and provisions at 1:1; a module submitting work to a service
|
||||
is neither. It is not an event, because an event is a broadcast nobody is obliged to act on and a
|
||||
second holder would do the work twice. It is not a provision, because there is no resource and no
|
||||
credential. What makes it safe is not cleverness in the subscribe call but the seat: exactly one
|
||||
holder, so exactly one worker, by construction.
|
||||
|
||||
**State** is the shape the deploy path needs and nothing else uses. A declaration is not an event
|
||||
— replaying yesterday's is actively harmful — and not a job. Only the newest matters, which is
|
||||
last-per-subject retention, and a node that has seen sequence *n* refuses *n−1* by construction.
|
||||
That is the wire-level answer to
|
||||
[issue 107](../../04-ISSUES/107-a-declaration-carries-no-order/00-report.md).
|
||||
|
||||
## 5. Seats
|
||||
|
||||
A module declares a seat with its protocol, and the mesh enforces one holder at its scope
|
||||
([ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md)). A caller declares that
|
||||
it uses the *seat*, never the module, so the implementation can be replaced under it.
|
||||
|
||||
- The set of seats is **derived** — the mesh's own, plus every registered module's — so it is both
|
||||
closed and extensible, and enumerating it is a query rather than an inventory.
|
||||
- `mesh-*` is **reserved**: the prefix is the reservation rule, and a module declaring one is
|
||||
refused at registration.
|
||||
- Two modules declaring the same name: the second is refused.
|
||||
- A module may not claim a seat whose protocol it does not implement.
|
||||
- **Nobody holding a seat is not an error.** The stream exists from registration, so work queues
|
||||
until a holder appears. Install the telegram module a week later and the backlog flushes.
|
||||
|
||||
## 6. The lifecycle: build, publish, deploy
|
||||
|
||||
Every shape above appears once, in order, and no step knows where the next one runs.
|
||||
|
||||
**A change lands.** The module holding `mesh-git` emits `pushed` — repository, ref, commit. An
|
||||
event, because it is a fact about git and git's identity is the meaning.
|
||||
|
||||
**The change becomes work.** The controller consumes `pushed`, asks the catalogue which modules
|
||||
are built from that repository and path
|
||||
([ADR 0069](../../02-DECISIONS/0069-a-module-is-a-repository-and-a-path.md)), and submits one
|
||||
**job** per affected module to the `mesh-build-machine` seat. The builder stays simple: it builds
|
||||
what it is handed, and never resolves anything. A builder that dies mid-build has its job
|
||||
redelivered, because a work queue with explicit ack is what that means.
|
||||
|
||||
**The artifact is published.** The builder pushes to the registry seats and emits `built` —
|
||||
module, version, digest. An event again: a fact about the builder.
|
||||
|
||||
**The build cascade is that event fanning out through a graph the mesh already has.** A module
|
||||
whose image is built *on* another's artifact declares that in `build.on`. So `built` reaches the
|
||||
controller, which walks the declared graph and submits rebuild jobs for everything downstream. A
|
||||
dependency cascade is not special machinery — it is one event, one derived graph, and the same job
|
||||
queue.
|
||||
|
||||
**Deployment is state, not a message.** The controller composes each affected node's declaration
|
||||
and publishes it last-per-subject. A node that was away gets exactly the current one, never a
|
||||
queue of superseded ones, and a replayed older one is refused by sequence.
|
||||
|
||||
**A version prepares its state before it runs.** *Built 2026-09-28.* A module version may declare an
|
||||
entrypoint that brings its state to the shape that version needs — the same vocabulary as the entrypoints it declares for its
|
||||
tools and its provisioner, and nothing about how a machine runs it. The mesh runs that entrypoint as it
|
||||
runs the module's own code, to completion, in the module's own context, and a version whose preparation
|
||||
did not succeed does not run: the step gates that module and nothing else on the machine
|
||||
([ADR 0136](../../02-DECISIONS/0136-a-step-gates-its-module-not-the-machine.md)), and the rollout stops
|
||||
at the first machine that did not take it
|
||||
([ADR 0135](../../02-DECISIONS/0135-a-module-version-prepares-its-state-before-it-runs.md), superseding
|
||||
[ADR 0133](../../02-DECISIONS/0133-a-module-owns-its-migrations-and-the-mesh-owns-when-they-run.md)).
|
||||
Once per state, and the mesh derives what a state is: a consumer is a module on a machine, so what the
|
||||
mesh provisions is per consumer and preparation is too. No level to choose, and no race to lock against.
|
||||
|
||||
**Applying is reported to a role.** The host applies and reports to the `mesh-controller` seat —
|
||||
not to an address it was given at genesis. Held and retried while the store restarts
|
||||
([ADR 0083](../../02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md)).
|
||||
|
||||
**And the mesh says what it applied.** *Built 2026-09-28.* A report is control traffic only the control plane reads, so the
|
||||
chain above went dark at the moment it touched a machine: nothing said which version a machine now runs,
|
||||
or that it refused to. The control plane states those as facts under its own seat's namespace, when what
|
||||
a machine runs changes rather than on every convergence pass, and anything that cares subscribes the way
|
||||
the catalogue subscribes to `built` ([ADR 0134](../../02-DECISIONS/0134-the-mesh-says-what-it-applied.md)).
|
||||
The facts are second-hand by design — one emitter, one ordering — and a machine that cannot reach the bus
|
||||
produces none, so absence is not health.
|
||||
|
||||
What disappears across that chain is every address. No webhook URL, no registered callback, no
|
||||
"which node is the builder on", no controller endpoint baked into a joining node. That is the
|
||||
class of bug
|
||||
[issue 102](../../04-ISSUES/102-an-address-recorded-at-genesis-or-build-does-not-follow-the-nodes-ports/00-report.md)
|
||||
names, dissolved rather than fixed.
|
||||
|
||||
## 7. Modules depending on each other
|
||||
|
||||
Three kinds, and conflating them is how deployment ordering goes wrong.
|
||||
|
||||
**Build-time** — A's image is built on B's artifact. Resolved by the cascade above; nothing at
|
||||
runtime cares.
|
||||
|
||||
**Provision** — A requires a database from B. A **hard** dependency: the credential must exist
|
||||
before A can start, so resolution gates delivery and A is shown as waiting until B has answered
|
||||
([design 27](27-a-module-requires-the-mesh-resolves.md)).
|
||||
|
||||
**Seat** — A uses B's seat. A **soft** dependency, and this is the one the bus changes. A starts
|
||||
whether or not anyone holds the seat, because the stream absorbs the gap. Deployment order stops
|
||||
mattering for everything expressed this way, and a service being restarted, moved or upgraded is
|
||||
not an outage for its callers — it is latency.
|
||||
|
||||
That difference is worth choosing on purpose. A dependency expressed as a provision must be
|
||||
ordered; the same dependency expressed as a seat need not be.
|
||||
|
||||
## 8. Versioning a protocol
|
||||
|
||||
A seat's protocol is a compatibility surface between modules that do not know each other and are
|
||||
deployed at different times. Four ways it can change, and they are not equally dangerous:
|
||||
|
||||
| change | example | detectable |
|
||||
|---|---|---|
|
||||
| **additive** | a new `accepts` subject, a new optional field | nothing breaks |
|
||||
| **removal or rename** | `send` becomes `deliver` | yes, mechanically |
|
||||
| **shape** | an optional field becomes required | yes, if shapes are specified |
|
||||
| **semantic** | `send` starts meaning *queue for tomorrow* | **no** |
|
||||
|
||||
**Additive is free.** A seat may grow without a version, without re-registering a caller, and
|
||||
without ceremony. Most change is this.
|
||||
|
||||
**A breaking change is refused while anyone is bound.** Registration computes a compatibility
|
||||
fingerprint over the seat's protocol — its subjects and the shapes they carry. A registration that
|
||||
alters the fingerprint while callers are bound is refused, **and the refusal names them**. The
|
||||
mesh already holds the `uses` graph, so this is derived rather than declared, and it turns a
|
||||
runtime breakage into a registration-time conversation.
|
||||
|
||||
**When a break is genuinely needed, the version goes in the subject, not the name.** The seat stays
|
||||
one thing; `mesh.seat.<seat>.v2.<verb>` runs beside v1 and the holder serves both. A caller moves
|
||||
when it is ready. Versioning the *seat name* was considered and rejected: it forks the role, so
|
||||
"one holder" stops meaning one provider of the capability, and every document naming the seat has
|
||||
to be found and changed.
|
||||
|
||||
**Binding is recorded, not inferred.** A caller declares `uses: telegram-sender` with no version,
|
||||
and resolution binds it to the current one and records that — the same **pin** machinery
|
||||
[design 27](27-a-module-requires-the-mesh-resolves.md) already uses when resolution had a choice
|
||||
to make. Moving to v2 is a deliberate re-pin, so nothing drifts onto a new protocol because it
|
||||
happened to be newest.
|
||||
|
||||
**Retirement is reported, never automatic.** When the `uses` graph shows nothing bound to v1, the
|
||||
overview says it is retirable. The mesh does not remove it.
|
||||
|
||||
**And none of this catches a semantic change.** Same subject, same shape, new meaning: no
|
||||
fingerprint sees it, and no check proposed here would. The defences are review, and pushing
|
||||
meaning into shape wherever it can go — a required `channel` field is caught, a changed
|
||||
interpretation of an existing one is not. Saying so is better than implying the fingerprint is
|
||||
complete, because a team that believes it is complete stops reviewing for the case it misses.
|
||||
|
||||
## 9. Provisioning over the bus
|
||||
|
||||
Provisioning rides the bus, and the provider stops having an address.
|
||||
|
||||
| part of a provision | shape |
|
||||
|---|---|
|
||||
| the requirement resolving to a provider | the controller's, not the bus's |
|
||||
| the grant reaching the provisioner | request/reply to a **role** |
|
||||
| `holds` — the reconcile question, every minute | the same call, on a timer |
|
||||
| `provisioned` / `deprovisioned` | events |
|
||||
| the credential reaching the consumer | §10 — fetched, never carried |
|
||||
|
||||
A provisioner's interface is already three calls — create, remove, holds — which is exactly a
|
||||
`serves` protocol. So **a provision interface is a seat whose protocol is those three**, which is
|
||||
why [design 26](26-the-seats.md) already allows a seat to deliver a provision. The two concepts
|
||||
were converging before this document; here they meet.
|
||||
|
||||
What stays different, and must not be unified away: a provision has a **per-consumer resource and
|
||||
a sealed credential**, created and destroyed per consumer. A seat protocol has neither — it is a
|
||||
role you send to. Collapsing them would mean pretending a database is a subject.
|
||||
|
||||
### `mesh-bus` and `nats` are two interfaces, never one name
|
||||
|
||||
The mesh's own bus is **`mesh-bus`**, delivered by the `mesh-broker` seat and answered by the
|
||||
controller — because the bus's accounts are configuration rather than something a provisioner
|
||||
creates, so there is no provisioner process in the path and nothing waiting on a bus account in
|
||||
order to make bus accounts. A module that runs a NATS server of its own and offers it as a
|
||||
backing service provides **`nats`**, exactly as the deprecated broker provides `amqp`
|
||||
([ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md): AMQP is not a provision, and everything speaks to the `mesh-broker` seat)).
|
||||
|
||||
They are never the same name. A manifest saying `nats` could otherwise mean either the mesh's
|
||||
nervous system or a private queue, and the difference between those is the whole architecture.
|
||||
0119's rule decides which is legitimate: a private bus is a backing service, never a channel to
|
||||
another module.
|
||||
|
||||
### Where addresses survive
|
||||
|
||||
"Where is it?" is two different problems, and the bus solves one of them completely and the other
|
||||
not at all. Keeping them apart matters, because a reader who thinks the mesh no longer has
|
||||
addresses will believe a class of bug is fixed when it is untouched.
|
||||
|
||||
**Something that is on the bus: the address disappears.** A host reporting in used to need the
|
||||
controller's address — recorded somewhere, at some moment, and wrong as soon as anything moved.
|
||||
Now it publishes to `mesh.seat.mesh-controller.report` and the bus routes it to whoever holds the
|
||||
seat. Nothing anywhere records where the controller is, so nothing can record it *wrongly*. The
|
||||
same is true of the builder, the catalogue, the telegram sender. This class is not mitigated; it
|
||||
is gone, because the information is no longer stored.
|
||||
|
||||
**Something that is not on the bus: the address stays, exactly as before.** A module that requires
|
||||
a database does not reach postgres over NATS — it opens a postgres connection, because postgres
|
||||
speaks postgres and is not listening on any subject. Its credential contains a host and a port,
|
||||
and no amount of subject addressing changes that.
|
||||
|
||||
So the fix for that second class is unchanged and is not this document's:
|
||||
[ADR 0098](../../02-DECISIONS/0098-a-fact-a-provider-makes-at-first-start-is-fetched-from-it.md) —
|
||||
fetch the fact where it is used rather than storing a copy — which is what
|
||||
[issue 102](../../04-ISSUES/102-an-address-recorded-at-genesis-or-build-does-not-follow-the-nodes-ports/00-report.md)
|
||||
is actually about. The bus makes that class *smaller* by removing every mesh-internal address from
|
||||
it. It does not make it empty.
|
||||
|
||||
**And one address is irreducible: the bus's own.** A node has to know where the broker is before
|
||||
it can use subjects for anything, so that one cannot be a subject. It is the addressing equivalent
|
||||
of §10's bootstrap — the first thing cannot be found by the mechanism that finds everything else.
|
||||
|
||||
## 10. Secrets, and why they never enter a stream
|
||||
|
||||
Everything the vault does is request/reply to the `mesh-vault` role: mint, fetch, rotate. In that
|
||||
sense it is as much on the bus as anything else.
|
||||
|
||||
**The bus is not trusted with a secret, and does not need to be.** A secret is sealed to its
|
||||
recipient, so what crosses the bus is ciphertext only that recipient can open. The broker sees
|
||||
that a secret moved, and to whom — metadata, which is acceptable — and never a plaintext.
|
||||
|
||||
**But sealed is not enough on its own, because a stream persists.** A sealed secret written into
|
||||
a JetStream stream is a durable ciphertext sitting in the mesh's own storage, and the day a
|
||||
sealing key leaks, that stream is an archive rather than a moment. So:
|
||||
|
||||
- **A secret travels on core request/reply, never through a stream.** No persistence, no replay,
|
||||
nothing to exfiltrate later.
|
||||
- **A declaration names a secret; it does not carry one.** Declarations are the state shape, which
|
||||
*is* a stream — so the host fetches the secret from the vault at apply time, over the core path.
|
||||
That is [ADR 0098](../../02-DECISIONS/0098-a-fact-a-provider-makes-at-first-start-is-fetched-from-it.md)'s
|
||||
existing discipline — *fetched from it, not carried* — applied to the one payload where carrying
|
||||
it is worst.
|
||||
|
||||
**The bootstrap, which is circular and has a precedent.** The vault makes every secret
|
||||
([ADR 0113](../../02-DECISIONS/0113-the-vault-makes-every-secret.md)), including the bus's own
|
||||
passwords. The vault is a module, and a module needs a bus account, whose password the vault
|
||||
makes. Nothing can go first.
|
||||
|
||||
This is the shape [ADR 0067](../../02-DECISIONS/0067-genesis-is-a-pivot.md) already resolves for
|
||||
the control plane: **genesis is a pivot.** The controller mints the handful of foundation
|
||||
credentials itself, raises the store, the broker and the vault, and then the vault takes over and
|
||||
mints everything from there — the same move as raising a temporary control plane and reinstalling
|
||||
it as an ordinary module once the registry exists.
|
||||
|
||||
So there are exactly two things the normal path cannot make, both at genesis, both ending the
|
||||
moment the mesh can mint for itself: **the bus's own accounts** (§the bootstrap argument in
|
||||
[ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md) (superseded by [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md): AMQP is not a provision, and everything speaks to the `mesh-broker` seat) (superseding [ADR 0125](../../02-DECISIONS/0125-the-bus-is-the-only-broker.md)) — a provisioner is a module and
|
||||
needs an account before it can run) and **the vault's own credential**. Any third exception is a
|
||||
design failure, and naming these two is what makes a third one visible.
|
||||
|
||||
## 11. Open
|
||||
|
||||
**Semantic change has no mechanical defence** (§8). Recorded as open rather than solved, because
|
||||
it is the residue of a question the rest of §8 answers and the part a fingerprint cannot reach.
|
||||
|
||||
**Whether a module may declare a seat it does not itself claim** — the contract as one thing, the
|
||||
implementation as another, which is how two competing implementations would ever exist.
|
||||
|
||||
**Whether a container should have a readiness notion.** Only an action carries `verify`, so a step that
|
||||
must run once a service *answers* — seeding through its own API — cannot be declared at all. Named here
|
||||
because the steps above make the gap obvious, not because they caused it.
|
||||
|
||||
**Whether `consumes` naming another module couples too tightly.** It is kept here deliberately —
|
||||
an event's provenance is its meaning — but a consumer of `billing.order.placed` does depend on
|
||||
billing existing under that name.
|
||||
|
||||
## 12. How it is checked
|
||||
|
||||
- **A manifest holds no subject.** A catalogue test: no manifest contains a string matching the
|
||||
subject grammar. The rule is worthless if it is followed by convention.
|
||||
- **A preparation is given what the module is given.** A composition test: what the preparation
|
||||
entrypoint receives equals what the module's own code receives, asserted rather than written twice —
|
||||
which is the drift a hand-written step invites, three times over in the catalogue today.
|
||||
- **A convergence that changed nothing says nothing.** Two identical reports, one emitted fact: what is
|
||||
guarded against is a fact per minute per machine, which is a stream nobody reads.
|
||||
- **Permissions are exactly the three namespaces.** A composition test per module: the derived
|
||||
permission set equals what its declaration implies, and a hand-written addition to it fails.
|
||||
- **A sender cannot read the queue it writes to.** A bed: a module declaring `uses` is refused
|
||||
subscribe on that seat's inbound subject.
|
||||
- **One holder, one delivery.** A bed: a seat's job delivered once with the holder running, and
|
||||
a second claim of the seat refused.
|
||||
- **A queued job survives no holder.** A bed: submit with the seat unheld, assign the holder,
|
||||
the job is delivered.
|
||||
- **The cascade rebuilds exactly the dependents.** A bed: publish an artifact two modules build
|
||||
on, and exactly those two are rebuilt.
|
||||
- **A stale declaration is refused.** A bed: replay sequence *n−1* after *n*, and the node refuses
|
||||
it rather than applying it.
|
||||
@@ -0,0 +1,137 @@
|
||||
---
|
||||
layer: to-be
|
||||
status: designed
|
||||
code: []
|
||||
updated: 2026-09-28
|
||||
decisions:
|
||||
- 02-DECISIONS/0132-a-seat-carries-the-tools-its-holder-must-serve.md
|
||||
- 02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md
|
||||
- 02-DECISIONS/0095-the-control-plane-is-the-way-to-ask-a-module.md
|
||||
---
|
||||
|
||||
# 33 — The tools the mesh answers
|
||||
|
||||
**An agent can call the mesh's tools and cannot find out what they are.** Both halves were measured
|
||||
on the live mesh on 2026-09-28: a client holding an operator credential connected to the bus, asked a
|
||||
module for its repositories and got them; the same client's request for the tool list found nothing
|
||||
serving it. The transport works, the account model works, the adapter that speaks the agent protocol
|
||||
works. What is missing is the mesh being able to say what it can do.
|
||||
|
||||
This design is the answer to that question, and it has three families in it, because a tool belongs to
|
||||
whoever is accountable for answering it.
|
||||
|
||||
## 1. Three families, and why the split is not arbitrary
|
||||
|
||||
| Family | Addressed to | Where the definition lives | Example |
|
||||
|---|---|---|---|
|
||||
| A **role's** tools | the seat: `mesh.seat.<seat>.tool.<verb>` | the seat's protocol, in the mesh's records | ask *the forge* to list its repositories |
|
||||
| A **module's** tools | the module: `mesh.mod.<module>.tool.<name>` | that module's code | ask *this gitea* for `gitea_list_repos` |
|
||||
| The **mesh's** own verbs | the `mesh-controller` seat | the seat's protocol, as above | `status`, `push`, `build`, `assign` |
|
||||
|
||||
The split follows accountability. A role is something the mesh guarantees exactly one holder of, so
|
||||
what the role answers is the mesh's to define and a holder's to implement
|
||||
([ADR 0132](../../02-DECISIONS/0132-a-seat-carries-the-tools-its-holder-must-serve.md)). A module's
|
||||
own tools are nobody's business but the module's, and their definitions live where they are
|
||||
implemented, because a copy kept anywhere else drifts from the code that answers.
|
||||
|
||||
The mesh's own verbs are the third family only in where they come from, not in kind: the control plane
|
||||
holds a seat like anything else, and its tools are that seat's. This is what keeps them addressable
|
||||
while the control plane is being replaced, which is the moment they are most needed.
|
||||
|
||||
**Both names for one capability is deliberate and bounded to this.** A forge holding the `git` seat
|
||||
answers the role's `list_repos` and its own `gitea_list_repos`, because the same module may run
|
||||
without the seat — a second instance, kept for one purpose — and then only the second name is true.
|
||||
The caller chooses which question it is asking. Nothing else in the mesh gets two names.
|
||||
|
||||
## 2. What a seat's tool is
|
||||
|
||||
A verb, what it does, and the schema of its arguments and its answer. A name alone is not callable by
|
||||
something that has never seen the mesh before, which is the whole population this surface exists for.
|
||||
|
||||
The protocol a seat carries today is three lists of bare verbs
|
||||
([ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md)), and it must widen to
|
||||
carry the rest. Two constraints on that widening:
|
||||
|
||||
- **It lives in the mesh's records, not in the control plane's binary.** Today a seat's protocol comes
|
||||
from compiled defaults, merged in as a row is read, because the seat rows never gained the columns.
|
||||
Discovery that reads a binary is discovery that disagrees with the mesh the moment the two are on
|
||||
different versions.
|
||||
- **The schema is stated in the form an agent protocol already uses**, so nothing translates between a
|
||||
seat's idea of an argument and the caller's. A translation layer would be a second definition of
|
||||
what a tool is.
|
||||
|
||||
## 3. Holding a seat means serving its tools
|
||||
|
||||
A module may not occupy a seat unless it serves every verb that seat declares. This joins the
|
||||
conditions of holding that already exist — providing what the seat delivers, being assigned at the
|
||||
seat's scope — and is refused the same way: at registration and at handover, naming the verbs that are
|
||||
missing rather than the fact that something is.
|
||||
|
||||
A module knows which seats it claims, so knowing which tools it must serve is not a discovery problem
|
||||
for the module: the seat says, the module implements, and anything beyond that is its own.
|
||||
|
||||
## 4. Addressing a node-scoped seat
|
||||
|
||||
A seat's subject is flat today — `mesh.seat.<seat>.<kind>.<verb>` — which is correct for a seat the
|
||||
mesh has one holder of and wrong for the six node-scoped seats, where one subject would reach every
|
||||
machine's holder and the holders' queue group would hand the call to whichever answered first. A
|
||||
node-scoped seat's tool therefore carries the node it is asked of. Nothing about a mesh-scoped seat
|
||||
changes.
|
||||
|
||||
## 5. Discovery
|
||||
|
||||
**What a role answers is a read.** The seats and their protocols are records, so the list is a query
|
||||
against the mesh's own store: no call to a module in the path, nothing that has to be running, and an
|
||||
answer that stays true while a holder is restarting or being replaced.
|
||||
|
||||
**What a module answers comes from the module.** Its definitions live in its code, so it is asked, and
|
||||
the answer is as available as the module is — which is the right coupling for a tool that only exists
|
||||
while that module does.
|
||||
|
||||
A caller therefore gets one list assembled from two sources, and the difference is visible in it: a
|
||||
role's tool names a seat, a module's names a module. An agent that wants to survive a holder being
|
||||
replaced binds to the first.
|
||||
|
||||
## 6. What serves this to an agent
|
||||
|
||||
A module the mesh assigns to the machine where the agent runs, holding a credential the mesh minted,
|
||||
with authority derived from what it may call — not a program started by hand with a credential printed
|
||||
to a terminal. The adapter itself already exists and is thin by design; what changes is that it stops
|
||||
being something a person carries and becomes something the mesh runs, on a node, like everything else.
|
||||
|
||||
An agent's authority can then be role-shaped: *the forge's tools*, rather than a list of
|
||||
module-specific names that changes the day the forge is replaced.
|
||||
|
||||
## 7. Versioning
|
||||
|
||||
A seat's tools are an interface and change like one. Additive within a version. A change that would
|
||||
break a caller takes the version token the subject already has room for, and the two versions run side
|
||||
by side until nothing is bound to the old one.
|
||||
|
||||
## How it is checked
|
||||
|
||||
- **A holder missing a verb cannot take the seat.** One test per condition of holding, as the existing
|
||||
conditions have, and the live refusal names the verbs.
|
||||
- **A verb nobody declared is a subject nobody may use.** The bus grants are derived from the seat's
|
||||
protocol already, and the golden composition of the user list is what keeps that honest: a holder is
|
||||
granted exactly the seat's verbs, a user of the seat exactly the publish side.
|
||||
- **Discovery needs no running module.** The test for a role's tools reads records and asserts the
|
||||
answer equals what the seats declare — if it needed a module up, it would not be a read.
|
||||
- **Two nodes holding one node-scoped seat derive two addresses.** Checked by the same test as the rest
|
||||
of the subject table.
|
||||
|
||||
## What this does not settle
|
||||
|
||||
- Which verbs each seat should serve. That is a decision per seat, and the reason to do it slowly: a
|
||||
seat's tools bind every future holder.
|
||||
- Whether a module's own tool definitions should also be recorded when a build resolves its manifest.
|
||||
There is an argument for it — the mesh could then answer for a module that is down — and an argument
|
||||
against, which is that a recorded copy of a live definition is a copy that can be wrong.
|
||||
|
||||
## References
|
||||
|
||||
- [ADR 0132](../../02-DECISIONS/0132-a-seat-carries-the-tools-its-holder-must-serve.md) — the decision this designs
|
||||
- [ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md) — a seat carries the protocol of its role
|
||||
- [ADR 0095](../../02-DECISIONS/0095-the-control-plane-is-the-way-to-ask-a-module.md) — a tool call passes one process where an audit belongs
|
||||
- [`26-the-seats.md`](26-the-seats.md) — what a seat is, how it is held and handed over
|
||||
- [`25-the-bus-on-nats.md`](25-the-bus-on-nats.md) §7 — a person's account, their inbox, and the adapter
|
||||
@@ -35,11 +35,13 @@ document is written and this one's status becomes `implemented`.
|
||||
| [`23-choosing-a-provider.md`](23-choosing-a-provider.md) | Which of several providers of a kind serves a consumer, and when a module carries its own instead | [ADR 0084](../../02-DECISIONS/0084-which-provider-serves-a-consumer.md), [ADR 0027](../../02-DECISIONS/0027-a-provision-names-what-the-consumer-is-coupled-to.md) |
|
||||
| [`24-the-secrets-vault.md`](24-the-secrets-vault.md) | The module that owns a secret — a `secret` provision, and the boundary of what it owns | [ADR 0085](../../02-DECISIONS/0085-a-secret-is-a-provision.md), [ADR 0031](../../02-DECISIONS/0031-the-control-plane-authenticates-nobody.md), [ADR 0048](../../02-DECISIONS/0048-a-provider-creates-the-credential-the-mesh-minted.md) |
|
||||
| [`25-the-bus-on-nats.md`](25-the-bus-on-nats.md) | **Proposed.** The architecture of the mesh's bus on NATS: what rides which subject under which guarantee and whose account, how a node joins, how a person reaches a tool, and how the mesh moves from the bus it has | [ADR 0106](../../02-DECISIONS/0106-the-bus-is-nats.md), [ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md), [ADR 0043](../../02-DECISIONS/0043-a-module-broker-account-is-scoped-by-emits-and-consumes.md), [ADR 0083](../../02-DECISIONS/0083-one-push-leaves-the-mesh-consistent.md) |
|
||||
| [`26-the-seats.md`](26-the-seats.md) | **Proposed.** What a mesh can have one of, who fills each, and a seat's holder answering for the provision it delivers — including the `git` seat a build's source can live on | [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md), [ADR 0111](../../02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md), [ADR 0109](../../02-DECISIONS/0109-a-package-registry-seat-is-one-per-ecosystem.md) |
|
||||
| [`27-a-module-requires-the-mesh-resolves.md`](27-a-module-requires-the-mesh-resolves.md) | **Proposed.** One concept for everything a module needs: a requirement with a contract, answered by one of four kinds of provider, resolved at assignment or refused. Retires settings, placeholders, facts and paths in definitions | [ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md), [ADR 0113](../../02-DECISIONS/0113-the-vault-makes-every-secret.md), [ADR 0114](../../02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md), [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md) |
|
||||
| [`26-the-seats.md`](26-the-seats.md) | **Proposed.** What a mesh can have one of, who fills each, and a seat's holder answering for the provision it delivers — including the `git` seat a build's source can live on | [ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)), [ADR 0111](../../02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md), [ADR 0109](../../02-DECISIONS/0109-a-package-registry-seat-is-one-per-ecosystem.md) |
|
||||
| [`27-a-module-requires-the-mesh-resolves.md`](27-a-module-requires-the-mesh-resolves.md) | **Proposed.** One concept for everything a module needs: a requirement with a contract, answered by one of four kinds of provider, resolved at assignment or refused. Retires settings, placeholders, facts and paths in definitions | [ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md), [ADR 0113](../../02-DECISIONS/0113-the-vault-makes-every-secret.md), [ADR 0114](../../02-DECISIONS/0114-a-shared-credential-rotates-over-two-credentials.md), [ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) (superseding [ADR 0110](../../02-DECISIONS/0110-a-seat-is-a-module-assignment-from-a-closed-set.md)) |
|
||||
| [`28-building-the-bus.md`](28-building-the-bus.md) | **Proposed.** The five steps of the bus work in the order their dependencies allow, each ending at a bed — with the surface measured, so no step's size is a guess | [ADR 0116](../../02-DECISIONS/0116-the-bus-is-built-in-five-steps.md), [ADR 0106](../../02-DECISIONS/0106-the-bus-is-nats.md), [ADR 0074](../../02-DECISIONS/0074-the-wire-is-specified-not-the-types.md) |
|
||||
| [`29-a-node-has-operator-accounts.md`](29-a-node-has-operator-accounts.md) | **Proposed.** The mesh models machines but not the humans on them: a node gains operator accounts, and a resource may live under a home owned by its account — what would own ~/.ssh, dotfiles and ~/.config when HAL retires | [ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md), [ADR 0051](../../02-DECISIONS/0051-shared-data-is-the-operators.md) |
|
||||
|
||||
| [`32-what-a-module-declares.md`](32-what-a-module-declares.md) | **Proposed.** What a module declares and what the bus derives from it: three namespaces, subjects from local names, queues never declared, the five relationships, and the build-publish-deploy lifecycle on one bus | [ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md), [ADR 0127](../../02-DECISIONS/0127-amqp-is-a-provision-not-the-bus.md), superseded by [ADR 0131](../../02-DECISIONS/0131-everything-on-the-mesh-speaks-to-the-broker-seat.md) (superseding [ADR 0125](../../02-DECISIONS/0125-the-bus-is-the-only-broker.md)), [ADR 0041](../../02-DECISIONS/0041-events-are-a-relationship.md) |
|
||||
|
||||
## Not yet written
|
||||
|
||||
- **The remaining six contexts.**
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
---
|
||||
status: fixed
|
||||
status: resolved
|
||||
opened: 2026-09-24
|
||||
located-in: [mesh-controller internal/inventory, mesh-controller internal/catalogue, mesh-controller cmd/mesh-controller, mesh-catalog modules/dnsmasq]
|
||||
fixed-by: [mesh-controller#73 overlay-name + namesInTheMesh, mesh-catalog#106 dnsmasq daemon.json merge]
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-27
|
||||
located-in: [mesh-catalog modules, mesh-control internal/catalogue, mesh-tools src]
|
||||
fixed-by: mesh-catalog 7b06a7a, mesh-tools fbeb373, mesh-control 05ff606
|
||||
amended-design: 03-DESIGN/01-to-be/32-what-a-module-declares.md
|
||||
---
|
||||
|
||||
# 127 — A module's event derives a subject nothing publishes
|
||||
|
||||
## What was observed
|
||||
|
||||
[Design 29](../../03-DESIGN/01-to-be/32-what-a-module-declares.md) §1 says a module names an event
|
||||
locally and the mesh derives the subject: `emits: order.placed` becomes
|
||||
`mesh.mod.<module>.event.order.placed`, and a consumer declaring `consumes: shop.order.placed`
|
||||
subscribes the emitter's own subject. That derivation is built and tested.
|
||||
|
||||
**Every event name in the catalogue is still written the way a routing key on the bus the mesh has
|
||||
is written** — `module.<module>.<verb>` — and the derivation reads it as `<emitter>.<event>`. Asked
|
||||
of the composer directly, with the module names and declarations the catalogue holds today:
|
||||
|
||||
| declared | derived |
|
||||
|---|---|
|
||||
| `builder` emits `module.builder.built` | publish `mesh.mod.builder.event.module.builder.built` |
|
||||
| the catalogue consumes `module.builder.built` | subscribe `mesh.mod.module.event.builder.built` |
|
||||
| a media module emits `module.<itself>.download.completed` | publish `mesh.mod.<itself>.event.module.<itself>.download.completed` |
|
||||
| a player consumes `module.*.download.completed` | subscribe `mesh.mod.module.event.*.download.completed` |
|
||||
|
||||
The consumer's subject names a module called `module`. **No cross-module subscription in the
|
||||
catalogue matches what any emitter publishes.** Thirty-seven manifests declare events; every one of
|
||||
their consume declarations derives this way.
|
||||
|
||||
Two further consequences of the same cause, found in the same check:
|
||||
|
||||
- One module declares `consumes: "#"` — the wildcard of the bus the mesh has, which is not a
|
||||
subject at all. The composer **refuses it outright**, so that module's account cannot be composed
|
||||
and the module cannot be assigned.
|
||||
- One module emits under a name that is not its own — it declares `module.<other>.image.pushed`
|
||||
while being a differently named module — which the derivation puts inside *its* namespace. Whether
|
||||
that is legitimate is a design question: design 32 §2 makes an event's source a fact the server
|
||||
enforces, and this is a module claiming another's name in its own event.
|
||||
|
||||
None of it fails on the bus the mesh runs on today, where a routing key is matched literally and
|
||||
nothing derives anything. It fails only once the subject is derived — which is to say it fails on
|
||||
the first mesh raised on the new bus, and not before.
|
||||
|
||||
Evidence: run against the controller's own `PermissionsFor` on the current feature branch, with the
|
||||
declarations read from the catalogue's manifests. Found while wiring the controller's consume side
|
||||
(design 28 step 3.4), when the controller's own subscription had to be written and the subject it
|
||||
would have to name turned out not to be the one design 32 specifies.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**This is the failure [ADR 0074](../../02-DECISIONS/0074-the-wire-is-specified-not-the-types.md)
|
||||
exists to catch, arriving by a route the conformance suite does not cover.** Two implementations
|
||||
that disagree about an envelope do not fail to compile — they ignore each other while both keep
|
||||
running. Here it is not two implementations disagreeing but a *declaration* and a *derivation*
|
||||
disagreeing, and the symptom is identical: every service starts, every log is quiet, and nothing
|
||||
reacts to anything.
|
||||
|
||||
The fixtures cannot catch it. They pin one emitter's envelope against one subject, and both halves
|
||||
of that pair are correct. What is wrong is only visible when an emitter's derived subject is set
|
||||
beside a consumer's derived subject — a check nothing performs, because until the subject was
|
||||
derived there was nothing to compare.
|
||||
|
||||
It also means the rule 3.8 established is weaker than it reads. That task asserted **no manifest
|
||||
contains a subject**, which holds: a manifest contains a local name. What nothing asserts is that a
|
||||
local name derives to a subject some emitter actually publishes, and the rule as stated is satisfied
|
||||
by thirty-seven manifests whose names derive to nothing.
|
||||
|
||||
And it blocks work already scheduled. Design 28's step 4.2 (a build source's change reaching the
|
||||
builder over the bus) and 4.3 (an installation completing over the bus) are both event flows through
|
||||
exactly these pairs, and the catch-up flow the controller answers is a third — the controller
|
||||
currently replays a build announcement under its *own* name rather than the builder's, which a
|
||||
consumer filtering the builder's subject will not hear either.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Is a local name converted per manifest (`emits: built`), or does the derivation keep accepting the
|
||||
old form and strip a redundant prefix? The first is thirty-seven manifests and a rule that can be
|
||||
checked; the second is a rule that cannot, because `module.foo.bar` is also a legitimate three-part
|
||||
local name.
|
||||
- What checks the pair? An emitter's derived subject against every consumer's derived subject is a
|
||||
whole-catalogue check, not a per-manifest one — and a module lives in its own repository and may
|
||||
be registered long after the catalogue was checked.
|
||||
- What are `#` and `*` in a consumed name? The bus the mesh has and the bus being built spell
|
||||
wildcards differently, and a `consumes` pattern is the one place a module writes one.
|
||||
- May a module emit an event named after another module, and if not, what does the module that does
|
||||
it today declare instead?
|
||||
- Who replays? A catch-up answer published by the controller under a builder's subject is the
|
||||
controller signing an event as another module, which is the thing the derived namespace prevents.
|
||||
If it must not, then a replay is a different message from an announcement, and the consumer needs
|
||||
to be told so.
|
||||
@@ -0,0 +1,109 @@
|
||||
# Diagnosis — 2026-09-27
|
||||
|
||||
## Where it lives
|
||||
|
||||
Three places, and only one of them is a bug in code.
|
||||
|
||||
**The manifests, in the module catalogue.** Thirty-seven declare events, and every one of them
|
||||
spells an event the way a routing key on the bus the mesh runs on today is spelled —
|
||||
`module.<module>.<verb>`. [Design 29](../../03-DESIGN/01-to-be/32-what-a-module-declares.md) §1 says
|
||||
a module names an event **locally and bare** (`emits: order.placed`) and a consumer names
|
||||
`<emitter>.<event>` (`consumes: billing.order.placed`). So the manifests are stale against a rule
|
||||
that was already decided, not wrong against an undecided one. **This is the whole of the reported
|
||||
symptom.**
|
||||
|
||||
**The manifest's own documentation, in the parser.** The comments on `emits` and `consumes` still
|
||||
describe the old convention and give the old examples — "dotted topic keys, e.g.
|
||||
`module.umami.site.created`", and `"#"` named as the audit logger's pattern. A module author reading
|
||||
the file they read most is being told to write the thing that does not work. That is why the drift
|
||||
was uniform across thirty-seven manifests rather than scattered: nobody was mistaken, everyone
|
||||
followed the documentation.
|
||||
|
||||
**Nothing checks either one.** `ParseManifest` validates the module name, the slug, what it provides
|
||||
and what it requires. It says nothing about an event name. So a local name that derives to a
|
||||
namespace belonging to a module called `module` is accepted by every check the mesh has, and the
|
||||
first thing that notices is a subscription that never fires.
|
||||
|
||||
## What was ruled out
|
||||
|
||||
**The derivation is not wrong.** Asked directly, with the module names and declarations the
|
||||
catalogue holds, `PermissionsFor` produces exactly what design 32 §1 specifies for the input it is
|
||||
given: it reads a consumer's `<emitter>.<event>` and builds the emitter's subject. Given
|
||||
`module.builder.built` it reads the emitter as `module`, which is a correct reading of an incorrect
|
||||
declaration.
|
||||
|
||||
**The conformance fixtures are not at fault and could not have caught it.** They pin one emitter's
|
||||
envelope against one subject, and both halves of that pair are correct. What is wrong is only
|
||||
visible when an emitter's derived subject is set beside a *consumer's* derived subject — a
|
||||
comparison nothing performs, because until the subject was derived there was nothing to compare.
|
||||
|
||||
**Task 3.8's rule is not broken, it is weaker than it reads.** That task asserted **no manifest
|
||||
contains a subject**, which holds: a manifest contains a local name. Nothing asserts that a local
|
||||
name derives to a subject some emitter actually publishes.
|
||||
|
||||
## What is still a decision and not a conversion
|
||||
|
||||
Converting the manifests is implementing design 29, not deciding anything. Three of the report's
|
||||
open questions are not:
|
||||
|
||||
- **Wildcards.** Design 29's table has no wildcard row, and two manifests need one: a module
|
||||
consuming every download completion across several media modules, and an audit logger consuming
|
||||
everything. The two buses spell wildcards differently, and a `consumes` pattern is the one place
|
||||
a module writes one.
|
||||
- **A module emitting under another module's name.** One manifest declares an event named for a
|
||||
*provision* rather than for itself. Design 29 §2 makes an event's source a fact the bus enforces,
|
||||
so this cannot survive as written — and the remedy is probably not a rename but a **seat**, which
|
||||
is what a name stable across whoever implements it already is.
|
||||
- **Who replays a build announcement.** The controller answers a catalogue's catch-up by
|
||||
re-publishing builds under its *own* name, which no consumer of the builder's subject hears, and
|
||||
for which it holds no grant. Publishing them under the builder's subject would be the controller
|
||||
signing an event as another module — the exact thing the derived namespace prevents. So the
|
||||
catch-up is either a different message or a different mechanism, and that is a decision.
|
||||
|
||||
## Owners
|
||||
|
||||
`located-in` names the manifests and the parser. The replay question reaches the controller and the
|
||||
catalogue module together and is recorded above rather than in that field, because it is not where
|
||||
this symptom lives.
|
||||
|
||||
# Fixed — 2026-09-27
|
||||
|
||||
Converted, and the rule now has checks. What it took was larger than the report said, in two
|
||||
directions nobody had looked.
|
||||
|
||||
**The module code, not just the manifests.** Forty-three files pass an event name to `emit()` at
|
||||
runtime, and the runtime builds the subject from what it is handed. A converted manifest with
|
||||
unconverted code would have had the permission and the subject disagree — the same silence, one layer
|
||||
down.
|
||||
|
||||
**Both clients had to learn the mapping.** Each passed the name straight through, which was right on
|
||||
the bus the mesh runs on today only because modules were writing routing keys. So the old bus's client
|
||||
now turns a local name into `module.<emitter>.<event>` on the way out and back on the way in.
|
||||
**Without that, converting the modules would have broken the mesh that is actually running** — which
|
||||
is the opposite of what fixing this was for.
|
||||
|
||||
**The declaration and the handler spoke different vocabularies.** The key a module's handler saw was
|
||||
the event name alone, while its manifest names `<emitter>.<event>`. So a correct manifest produced a
|
||||
pattern that could never match. The subject already carries the emitter; the key names it now.
|
||||
|
||||
## The three open questions, answered
|
||||
|
||||
- **Wildcards**: `*` is one name, `**` is the rest, spelled the mesh's way and derived to each bus's
|
||||
own. `**` alone is every event, which is what the audit logger wanted and now says in one token.
|
||||
- **A module emitting under another's name**: not allowed, and the remedy is the seat rather than a
|
||||
rename — a role's name outlives whoever fills it. **Deferred in practice**: seats carry protocol in
|
||||
the manifest and in the permission model, and the shared library cannot publish on one, so the
|
||||
module that did this emits under its own name and its consumers carry that coupling. Worth a task
|
||||
when a seat's holder needs to emit.
|
||||
- **Who replays a build announcement**: still open, and narrowed. It cannot become a reply to the
|
||||
catalogue's inbox: answering a module's inbox needs `_INBOX.>`, which is the blanket grant
|
||||
[design 25](../../03-DESIGN/01-to-be/25-the-bus-on-nats.md) §4 refuses. So the remaining options are
|
||||
a subject the controller may publish and the catalogue may subscribe, or a durable consumer that
|
||||
starts at the beginning of the stream and removes the need to ask at all. Recorded on the work
|
||||
breakdown as the catch-up half of task 4.5 rather than here, because it is no longer this symptom.
|
||||
|
||||
## What it found while running
|
||||
|
||||
Two dangling subscriptions that predated this and nothing had reported: a module emitting an event its
|
||||
manifest never declared, which the new bus refuses outright, and a module waiting for an event nothing
|
||||
emits — a demo that could never be triggered, because only that module may publish under its own name.
|
||||
@@ -9,7 +9,7 @@ amended-design: 02-DECISIONS/0118-undeclaring-gives-a-unit-back-the-state-it-was
|
||||
|
||||
## What was observed
|
||||
|
||||
Reviewing the uplink modules ([ADR 0117](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md))
|
||||
Reviewing the uplink modules ([ADR 0125](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md))
|
||||
found that the host's `remove` path stops every `service` resource that is no longer declared:
|
||||
`SetServiceState(..., "stopped")`, reported as "stopped; the unit file is not the host's to
|
||||
delete". `store.Orphans` matches by id alone. So any of these stops the unit:
|
||||
@@ -45,7 +45,7 @@ reaches it by.
|
||||
|
||||
## Resolution
|
||||
|
||||
[ADR 0118](../../02-DECISIONS/0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md):
|
||||
[ADR 0126](../../02-DECISIONS/0118-undeclaring-gives-a-unit-back-the-state-it-was-found-in.md):
|
||||
undeclaring removes what the mesh made and gives back what it changed. The host records the state
|
||||
it first found a unit in, and undeclaring returns the unit to it — a unit found running (the
|
||||
container runtime, sshd, a network manager) is left running; one the mesh started (the packet
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-27
|
||||
located-in: [mesh-catalog modules/gitea, mesh-controller cmd/mesh-controller, mesh-controller internal/builder]
|
||||
fixed-by: mesh-catalog #124 — the forge module watches for merged pull requests and emits `pull.merged` with the merge commit and the clone address; mesh-controller #110/#111 — the control plane follows that event on the bus, marks every module built from that repository and branch as moved, and builds them bases first, stopping when a base fails; mesh-controller #113/#114 — a build records the bases it was handed and the graph is read from builds, without which "bases first" had no edges to order by.
|
||||
amended-design: 03-DESIGN/01-to-be/28-building-the-bus.md
|
||||
---
|
||||
|
||||
# 131 — Nothing tells the mesh a source moved, and it reports itself current anyway
|
||||
|
||||
## What was observed
|
||||
|
||||
Six changes were merged to the trunk of six repositories in one sitting. The build machine built
|
||||
nothing. Its last build, minutes before the first merge, was still the one it reported; no build was
|
||||
requested, refused or failed, because none was ever asked for.
|
||||
|
||||
Asked afterwards what was wrong, the mesh said:
|
||||
|
||||
> 4 machine(s), all doing what they were told, all heard from, running what the mesh would send them,
|
||||
> and every module current with its source
|
||||
|
||||
Every one of the six had moved. The last clause was false for all of them, and it is the clause a
|
||||
person reads to decide whether there is anything to do.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The mesh learns a source moved by being told, and there is no longer anything to tell it.** The
|
||||
command exists — a person names the module and the commit — and so does the question the overview
|
||||
answers. What is missing is whatever used to connect the two. One repository still carries a forge
|
||||
webhook aimed at a port; the rest carry none, and the port belongs to a different service than the
|
||||
one the arrangement implies. So the state is not "the trigger is broken" but "there is no trigger,
|
||||
and nothing says so".
|
||||
|
||||
**A wrong answer is worse here than no answer.** "Every module current with its source" is
|
||||
indistinguishable, to a reader, from a mesh that has genuinely caught up. The overview is built to be
|
||||
the thing you check instead of checking by hand, so a confident false negative removes the habit that
|
||||
would otherwise have caught it. Nothing in the mesh is at fault for being out of date — it is at
|
||||
fault for saying it is not.
|
||||
|
||||
**It is also why "current with its source" cannot be a stored fact.** The mesh compares what it built
|
||||
against what it was last told the source was, and calls that agreement. Two facts agreeing tells you
|
||||
nothing when both come from the same place.
|
||||
|
||||
## The intended shape, which is decided and not built
|
||||
|
||||
The forge emits what happened to it — a pull request merged — and the build machine reacts by
|
||||
building what that commit affects. That keeps the forge ignorant of the build system and the build
|
||||
machine ignorant of the forge's internals, which is the same argument
|
||||
[ADR 0126](../../02-DECISIONS/0126-a-module-declares-its-own-seats.md) makes for addressing an event
|
||||
to its emitter: the merge is a fact about the forge, and what should be rebuilt because of it is not
|
||||
the forge's business to know.
|
||||
|
||||
The forge's module already declares the event. The build machine declares that it consumes nothing.
|
||||
|
||||
## What the trigger cannot be
|
||||
|
||||
**Not one build per changed module.** The modules form a graph: several are built from one
|
||||
repository, and some are the base another is compiled on — a runtime image, a compiler base, a
|
||||
repository whose context a second module builds from. Firing a build for each changed module
|
||||
independently would start work that cannot succeed yet and produce a failure per dependent, for one
|
||||
cause.
|
||||
|
||||
Observed while catching the mesh up by hand on 2026-09-27: a compiler base had to move before
|
||||
anything compiled against it could build, and when it failed, the right behaviour was for its
|
||||
dependents to wait rather than each fail the same way. Fifteen modules shared the cause. A trigger
|
||||
that reports it fifteen times has buried it.
|
||||
|
||||
So whatever reacts to the forge's event resolves what changed into an order, builds the bases first,
|
||||
and holds a dependent while its base is unbuilt or failed. That is a larger thing than "rebuild what
|
||||
the commit touched", and knowing it now is cheaper than discovering it from fifteen identical
|
||||
failures.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Is "the source moved" still a thing a person can assert by hand once the event path exists, or does
|
||||
the hand-operated form become the thing that made this failure possible?
|
||||
- Which commit does the build machine act on — the merge, or each commit it brought — and what does
|
||||
it do when several arrive for one module at once?
|
||||
- How does the overview stop being able to lie? Comparing what was built against what was recorded
|
||||
will always agree. Whether the trunk has moved is a question only the forge can answer, so either
|
||||
the overview asks it, or it stops claiming to know.
|
||||
- Does this want to be the same mechanism as the build request on the bus
|
||||
([ADR 0129](../../02-DECISIONS/0129-a-seat-carries-the-protocol-of-its-role.md)), or does it sit in
|
||||
front of it?
|
||||
|
||||
## What was done (2026-09-28)
|
||||
|
||||
The shape above was built as described: the forge's module emits the merge, the control plane
|
||||
consumes it, and nothing on either side knows the other's internals. The build is asked for the
|
||||
merge commit, not each commit the merge brought — the trunk moved once, to one place. Several merges
|
||||
for one module arriving in a row are followed in turn, each moving the recorded source to its own
|
||||
commit, so the last one to arrive is the one the mesh ends up built from.
|
||||
|
||||
The hand-operated form stays. `module moved` is how a source is recorded without a forge — a module
|
||||
built from a repository elsewhere, or a mesh whose forge module is down — and it is the same act the
|
||||
event performs, so the two cannot disagree about what "moved" means.
|
||||
|
||||
**Bases first needed edges, and there were none.** The order this report asked for was written and
|
||||
walked a graph that no build had ever recorded: a recipe reads its base from a build argument, so the
|
||||
digest was never in the file the builder read edges from. A build now reports what it was handed, the
|
||||
control plane records it by artifact path, and the order is read from each module's newest build.
|
||||
|
||||
**What still can lie.** The overview compares what was built against where it was last told the
|
||||
source is; the forge's event is now what moves that mark, so it is right for as long as the forge
|
||||
module was listening. A merge made while that module was down is a merge the mesh does not know of
|
||||
until the module next polls — it announces what merged since it last looked, so the gap closes when
|
||||
it comes back, and not before. The overview does not say so.
|
||||
@@ -0,0 +1,60 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-controller cmd/mesh-controller]
|
||||
fixed-by: mesh-controller — `module add` takes `--path` and `--self`, so a module handed over by hand records the whole location it came from; a record naming a repository and no directory says so in the reply; the rule is one function with a test beside it. The nine records already wrong were corrected by rebuilding each with its real directory, which is the same act through the same door.
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 132 — A module can be recorded without the directory it lives in
|
||||
|
||||
## What was observed
|
||||
|
||||
Nine modules on one mesh could not be rebuilt. Each attempt failed the same way:
|
||||
|
||||
> has no module.json at its root, so there is nothing saying what it is
|
||||
|
||||
All nine were recorded as coming from a repository that holds many modules, each in its own
|
||||
directory — and each record named the repository and no directory. So every build cloned the
|
||||
repository and looked for a manifest where there has never been one.
|
||||
|
||||
The failure only surfaced when something asked for all of them at once. Before that, the overview
|
||||
said every module was current with its source, because what it compares is what was built against
|
||||
what the mesh was last told the source has, and neither half knows whether the source can be found
|
||||
at all.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**A module is a repository and a directory inside it** ([ADR 0069](../../02-DECISIONS/0069-a-module-is-a-repository-and-a-path.md)),
|
||||
and one of the two doors into the catalogue could record only the first half. A build records the
|
||||
directory it was given, so a module that arrived by being built is always whole; a module handed over
|
||||
by hand had no way to say where it lived, and the flag to say it did not exist. The rule was decided
|
||||
and enforced on one path out of two.
|
||||
|
||||
**Half a location reads exactly like a whole one.** Nothing in the record is empty in a way a person
|
||||
would notice: the repository is there, the branch is there, the commit is there. The mesh only finds
|
||||
out at the moment it needs the manifest, which is the moment it is trying to rebuild — and the module
|
||||
stays on whatever it last built, indefinitely, with nothing saying why.
|
||||
|
||||
**It is the same shape as [131](../131-nothing-tells-the-mesh-a-source-moved/00-report.md).** A
|
||||
comparison between two facts the mesh holds about itself will agree with itself. Whether the source
|
||||
can be found is a question only an attempt to read it answers, and the answer had nowhere to go.
|
||||
|
||||
## What was done
|
||||
|
||||
`module add` takes the directory and which forge holds the repository, so a hand-registered module
|
||||
records the same whole location a built one does. What a record must say to be worth anything is one
|
||||
function with a test beside it, rather than a paragraph in a help string: provenance together or not
|
||||
at all, a directory needs a repository to be inside, a path on the mesh's own forge is not an address.
|
||||
And a record that names a repository but no directory says so when it is made — not refused, because a
|
||||
module really at a repository's root is ordinary, but said, because the person adding it is the one
|
||||
who knows which it is.
|
||||
|
||||
The nine wrong records were corrected by building each with its real directory, which re-records it.
|
||||
No row was written by hand.
|
||||
|
||||
## What is still true
|
||||
|
||||
A directory that does not exist in the repository cannot be refused when the module is added: the
|
||||
control plane does not clone, and inventing a check there would mean it did. The first build says so
|
||||
plainly, which is one build rather than nine, and the record it leaves behind is right from then on.
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-controller module.json]
|
||||
fixed-by: mesh-controller — the control plane's module declares a run-once `migrate` step before its server, which is the shape ADR 0052 prescribes for exactly this. A step's record of having run is the digest of its declaration and the image is part of that digest, so a new build of the control plane re-runs it; and because a run-once step gates what the declaration places after it, a migration that fails stops the new server from starting at all rather than letting it run against a schema it does not have.
|
||||
amended-design: 03-DESIGN/01-to-be/32-what-a-module-declares.md
|
||||
---
|
||||
|
||||
# 133 — The control plane's schema is migrated at birth and never again
|
||||
|
||||
## What was observed
|
||||
|
||||
On 2026-09-28 at 08:17 the control plane was replaced, by the mesh's own upgrade path, with a build
|
||||
whose code writes a column that a migration **in that same build** creates. Nothing ran the migration.
|
||||
|
||||
For the next three quarters of an hour the mesh built things and recorded none of them. Every build
|
||||
answered:
|
||||
|
||||
> ERROR: column "built_contexts" of relation "build" does not exist (SQLSTATE 42703)
|
||||
|
||||
and that sentence went only to whoever happened to be waiting on a build's reply. The overview kept
|
||||
saying the mesh was fine. The builds themselves worked — images were built and published — so the
|
||||
registry filled up with artifacts the mesh has no record of, and the graph stopped learning without
|
||||
anything saying so.
|
||||
|
||||
The schema was created once, at genesis, by an action in the foundation bundle that runs the same
|
||||
binary's `migrate`. Nothing runs it again. The mesh has updated its own control plane many times since
|
||||
that bundle, and every one of those updates carried whatever migrations the new build brought and
|
||||
applied none of them. This is the first time a build needed one.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The schema and the code that needs it ship as one artifact and are applied by two mechanisms, only
|
||||
one of which is automatic.** A module's version is atomic everywhere else in the mesh — the manifest,
|
||||
the image and what the machine runs move together. Its schema did not, so "the mesh updates itself on
|
||||
a push" was true of the code and false of what the code needs.
|
||||
|
||||
**The failure is quiet exactly where quiet is worst.** A build that cannot be recorded is a build that
|
||||
happened and left no trace, which is the fault [issue 050](../050-the-catalogue-knows-nothing-built-before-it/00-report.md)
|
||||
and [issue 131](../131-nothing-tells-the-mesh-a-source-moved/00-report.md) are both about. The mesh
|
||||
has three mechanisms for noticing a module is behind its source and none for noticing that what it
|
||||
recorded was refused.
|
||||
|
||||
**The shape was already decided, and the control plane was the one module that did not use it.**
|
||||
[ADR 0052](../../02-DECISIONS/0052-a-step-that-runs-once-before-a-container.md) says a run-once container is a
|
||||
step the host runs to completion before whatever the declaration places after it, and names migrating
|
||||
a schema as the case it exists for. The genesis code's own comment says a manifest may name its image
|
||||
in more than one resource — "a migrate step beside the server". The control plane's manifest had no
|
||||
such step; it went straight from a state directory to the server.
|
||||
|
||||
## What is still true
|
||||
|
||||
**Additive migrations are load-bearing, not a style preference.** The step runs before the *new*
|
||||
server starts, which means the old binary briefly runs against the new schema. A migration that
|
||||
removes or renames something would break the running control plane in the window between the two.
|
||||
|
||||
**A hand-written step is one the next module forgets**, which is why this fix is not where the matter
|
||||
ends: [ADR 0135](../../02-DECISIONS/0135-a-module-version-prepares-its-state-before-it-runs.md) makes it
|
||||
derived and puts it where an author works: a module version declares an entrypoint that prepares its
|
||||
state, and the mesh composes the gated work from it, so the control plane stops being the only module
|
||||
that had to remember. That record also settles the level question HAL answered with stages — a consumer
|
||||
is a module on a machine, so the scope of preparation is the scope of the state — and
|
||||
[ADR 0134](../../02-DECISIONS/0134-the-mesh-says-what-it-applied.md) answers the second open question
|
||||
below: what a machine applied, and what it refused, become facts on the bus rather than a line in a log.
|
||||
|
||||
**The mesh now has two shapes for one problem.** The catalogue module migrates its own schema in its
|
||||
own code when it starts; the control plane migrates in a step the host gates on. Both work and the
|
||||
reasons differ — a module that owns its store entirely can do it at start, while a step is visible in
|
||||
the declaration and refuses to let a broken upgrade serve. Which one the mesh should standardise on is
|
||||
a decision, not a fix, and it is not made here.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should a module be refusable at registration when it ships migrations and declares no step and no
|
||||
other way to apply them? The mesh can see both halves.
|
||||
- Should a record the store refuses reach the overview? Today the only reader of that failure is
|
||||
whoever asked for the thing that failed, and for an event arriving on the bus there is no such
|
||||
person.
|
||||
@@ -0,0 +1,66 @@
|
||||
---
|
||||
status: open
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-catalog, mesh-controller internal/catalogue]
|
||||
fixed-by:
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 134 — A definition may still name the mesh, and the check that would say so does not exist
|
||||
|
||||
## What was observed
|
||||
|
||||
[ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) says a module
|
||||
definition names no node, no mesh and no host path, and states how that is checked:
|
||||
|
||||
> A catalogue test finds no domain name in any definition value.
|
||||
|
||||
There is no such test. Run by hand on 2026-09-28, across the 72 manifests in the catalogue, the
|
||||
question it asks has 15 answers. They are not all the same kind of thing, and the difference matters
|
||||
more than the count:
|
||||
|
||||
**Values the mesh acts on** — seven:
|
||||
|
||||
| module | where | what it names |
|
||||
|---|---|---|
|
||||
| keycloak | `env.KC_HOSTNAME` | this installation's public name for itself |
|
||||
| minio | `env.MINIO_BROWSER_REDIRECT_URL` | the same, for its console |
|
||||
| invoicing | a resource's `image` | a named registry rather than the mesh's artifact store |
|
||||
| builder | `build.artifacts[].context.repository` | the forge, by URL |
|
||||
| route-proxy | `build.artifacts[].context.repository` | the forge, by URL |
|
||||
| route-adapter | a resource's `content` | a proxy's dynamic configuration |
|
||||
| novox.be | `module` | the module is named after the domain it serves |
|
||||
|
||||
**Prose** — eight, in `listens[].why`: de-spiegel, mailu, n8n, only-office, photos, photos-eef,
|
||||
photos-filip, portainer. Each explains what a port is for and mentions the public name it is reached
|
||||
by. Nothing reads these; a check written as a string search would report them, and reporting them as
|
||||
violations of the same rule would be wrong.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**An unenforced rule is indistinguishable from a wrong one, and costs more, because people believe
|
||||
it.** The record says the mesh is name-agnostic, four design documents rest on that, and a reader
|
||||
checking whether it holds finds that it does not — in the places that matter most. The two forge URLs
|
||||
are what a build reaches into for its source; the two hostnames are what a service tells a browser
|
||||
about itself.
|
||||
|
||||
**It is the difference between a mesh and this mesh.** A definition carrying `novox.be` is a
|
||||
definition that can only be installed here. The whole point of the rule is that the same catalogue
|
||||
raises a different mesh with a different name, and today seven modules would need editing to do it.
|
||||
|
||||
**And the shape of the fix is not the same for each.** A public name is an operator's choice about an
|
||||
assignment, which ADR 0112 already provides for; a forge URL should be a path on the git seat
|
||||
([ADR 0111](../../02-DECISIONS/0111-a-build-source-is-on-the-git-seat-or-external.md)); an image from a named
|
||||
registry is a question about the artifact store, not about naming. Counting them together would hide
|
||||
that.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Does a domain in a `why` string break the rule? It is documentation the mesh never reads, and a
|
||||
check that cannot tell the two apart will either pass things it should catch or fail things nobody
|
||||
should change.
|
||||
- Where does a service's public name live, concretely — a setting on the assignment, or a fact the
|
||||
mesh composes from the node's domain? ADR 0112 says a requirement the mesh resolves; the two
|
||||
hostnames above are the first real cases.
|
||||
- Should a build context name a repository on the git seat rather than by URL, and if so, what does
|
||||
that mean for a context in *another* mesh's forge?
|
||||
@@ -0,0 +1,70 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-host internal/apply]
|
||||
fixed-by: mesh-host — a container's mesh names are part of the spec digest the host compares, sorted so the digest does not move for a reordering. A container whose names moved is now recreated like a container whose image moved, and the test fails against the previous behaviour.
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 135 — A container's mesh names are not compared, so a moved address is never noticed
|
||||
|
||||
## What was observed
|
||||
|
||||
One container on this mesh had been restarting every thirty seconds for five days — 2286 times — and
|
||||
the mesh reported the machine as doing what it was told.
|
||||
|
||||
Its logs said its database connected and then a query timed out. The database was reachable: the same
|
||||
query from the same network, with the same credential, answered in milliseconds. What differed was the
|
||||
name. Inside that container, `novox.internal` resolved to `10.42.0.1`; in every other container on the
|
||||
machine it resolved to `10.10.0.1`. The mesh's overlay range had moved, and this container still held
|
||||
the old one:
|
||||
|
||||
```
|
||||
umami created 2026-09-23 novox.internal:10.42.0.1
|
||||
mesh-catalog created today novox.internal:10.10.0.1
|
||||
```
|
||||
|
||||
A container resolves other machines and public names through the entries the mesh gives it when it is
|
||||
created, and nothing re-reads them afterwards. The host compares a container against what was declared
|
||||
by a digest of its spec — image, name, environment, ports, volumes, arguments, resolver, address, and
|
||||
what it reads — and **the mesh's names were not in it**. So this container matched what was declared,
|
||||
was left alone, and kept an address that had not existed for five days.
|
||||
|
||||
Forty-eight other containers had current names. Not because anything corrected them: each had been
|
||||
recreated for some other reason — a new image, a changed file — and picked up the current roster on the
|
||||
way. This one's image is an upstream release that had not moved, and nothing else about it changed, so
|
||||
nothing ever recreated it.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**It is the exact fault [issue 045](../045-a-container-keeps-the-values-it-started-with/00-report.md)
|
||||
named, in the one field that was left out.** That issue is why the digest carries what a container
|
||||
reads: "a container whose configuration had since been rewritten compared equal and was left alone —
|
||||
running values the machine no longer holds, while every check reported success." The same sentence
|
||||
describes this, with *names* in place of *files*.
|
||||
|
||||
**The failure is invisible in exactly the way that matters.** The container runs, so the machine
|
||||
reports it applied. It restarts, but a restarting container is a normal sight during an upgrade. The
|
||||
only account of the fault is inside the container's own log, in the words of the application rather
|
||||
than of the mesh — and what it says is that a query timed out, which points at the database.
|
||||
|
||||
**And it is most likely to bite what changes least.** Every container that is rebuilt often repairs
|
||||
itself by accident. The victim is the module whose image is stable — which is to say, the module that
|
||||
was working fine.
|
||||
|
||||
## What was done
|
||||
|
||||
The mesh's names are part of the digest, sorted so the digest does not move for a reordering nobody
|
||||
made. A container whose names moved is now recreated exactly as one whose image moved.
|
||||
|
||||
The first apply after this recreates every container that carries mesh names — one restart each,
|
||||
already the price the mesh pays for any image update — because their recorded digests predate the
|
||||
field.
|
||||
|
||||
## What is still true
|
||||
|
||||
The mesh gives a container its names at creation and has no way to change them in place. That is the
|
||||
container runtime's shape, not a choice; the answer is to recreate, which is what this does. A module
|
||||
that would rather re-read a roster from a file can already ask for one as a fact
|
||||
([ADR 0120](../../02-DECISIONS/0120-a-roster-fact-carries-its-format-as-a-template.md)) and restart on
|
||||
it.
|
||||
@@ -0,0 +1,92 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-catalog modules/fail2ban]
|
||||
fixed-by: mesh-catalog — the intrusion-prevention module bans through an action it ships itself, already in use on every machine, instead of naming a firewall front-end two of them do not have. The instance is closed; the class in "What is still true" is not.
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 136 — A module may name a program the machine does not have, and everything reports success
|
||||
|
||||
## What was observed
|
||||
|
||||
Two machines were given the intrusion-prevention module on 2026-09-28. Both refused to start it:
|
||||
|
||||
```
|
||||
ERROR Failed during configuration: Have not found any log file for 'recidive' jail.
|
||||
ERROR Async configuration of server failed
|
||||
fail2ban.service: Main process exited, code=exited, status=255/EXCEPTION
|
||||
```
|
||||
|
||||
The jail that bans whoever keeps coming back reads the service's *own* log, and the service checks
|
||||
every jail's log file while it configures itself — before it has created that log. The module
|
||||
declared the jail and shipped the rotation for that log, and never declared the log. On the two
|
||||
machines where it had run for years the file was simply there, so nothing had ever noticed.
|
||||
|
||||
That failure was loud. Fixing it uncovered a second one in the same module that is not.
|
||||
|
||||
The module's defaults named `ufw` as the way to ban an address. Two of these four machines have no
|
||||
`ufw` — they filter with nftables — and nothing checks that until an address is banned. Asked to ban
|
||||
a documentation address on such a machine, the service accepted the instruction, counted it, ran the
|
||||
command, and wrote this to a log nobody reads:
|
||||
|
||||
```
|
||||
ERROR ... -- stderr: '/bin/sh: line 5: ufw: command not found'
|
||||
ERROR ... -- returned 127
|
||||
ERROR Failed to execute ban jail 'sshd' action 'ufw' ... Error banning 192.0.2.99
|
||||
```
|
||||
|
||||
No rule existed afterwards. Throughout, the unit was `active`, the module was applied, and the
|
||||
machine's report said so. **A machine had been added to the mesh's intrusion prevention, reported as
|
||||
protected, and was banning nobody.**
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The two faults are the same mistake with opposite symptoms.** Both are the module assuming
|
||||
something about the machine — a file that happens to exist, a program that happens to be installed.
|
||||
One stopped the service, which anybody notices. The other left it running and empty, which nobody
|
||||
does. A mesh that only catches the loud one is a mesh whose coverage is unknown.
|
||||
|
||||
**"The unit is running" was taken for "the module is doing its job".** That is the only health a
|
||||
service resource has. It is the right answer for most modules and it is silent for any module whose
|
||||
work happens later, on an event — a ban, a renewal, a backup, a notification. The report cannot
|
||||
distinguish "protecting this machine" from "installed and inert".
|
||||
|
||||
**And it is exactly the naming rule, one level down.**
|
||||
[ADR 0112](../../02-DECISIONS/0112-a-module-definition-names-no-node-mesh-or-path.md) says a
|
||||
definition names no node, no mesh and no host path, because the same definition has to raise a
|
||||
different mesh. `ufw` is not a node name, but it is the same class of assumption: a value the module
|
||||
cannot know, true on some machines and false on others, written as though it were a constant. The
|
||||
module already knew how to do better a few lines away — the mesh's own address range is named there
|
||||
as something the machine fills in.
|
||||
|
||||
## What was done
|
||||
|
||||
The module declares the log its own jail reads, created once and never touched again, since what
|
||||
grows in it is the service's and the rotation the module already ships is what keeps it small. And
|
||||
it bans through the action it ships itself, which every machine here can run, which was already in
|
||||
use by the other jail on all four, and which covers a container's published port as well as the
|
||||
host's own.
|
||||
|
||||
All four machines now run it, with both jails, and a ban lands on each — verified by banning and
|
||||
unbanning a documentation address on every one.
|
||||
|
||||
## What is still true
|
||||
|
||||
**Nothing would have caught either fault before it shipped.** The control plane reads a manifest, not
|
||||
a machine; `ufw` and `/var/log/…` are strings in a file it has no way to evaluate. The host could in
|
||||
principle be asked whether a declared program exists, but no resource says "this file names a command
|
||||
that must be there", so there is nothing to check.
|
||||
|
||||
**Two machines' bans from before this are stale rules in the old front-end**, which the service no
|
||||
longer knows about and will never lift. They reject two addresses for ever. Harmless, and a reminder
|
||||
that changing how a module enforces something leaves what it already enforced behind.
|
||||
|
||||
## Open questions
|
||||
|
||||
- What does a service resource's health mean for a module whose work is event-driven? A unit being
|
||||
active is the weakest claim available, and four of this mesh's modules are of that kind.
|
||||
- Should a declaration be able to say that a resource depends on a program, so the machine can refuse
|
||||
what it cannot carry out rather than reporting success?
|
||||
- Where should the packet filter a module bans through come from — the module's own choice, as now,
|
||||
or the seat that owns the machine's filtering?
|
||||
@@ -0,0 +1,74 @@
|
||||
---
|
||||
status: resolved
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-controller internal/catalogue]
|
||||
fixed-by: mesh-controller — a machine says which networks it routes and the derived filter forwards them, their guests keeping address and name service ([ADR 0137](../../02-DECISIONS/0137-a-machine-says-which-networks-it-routes.md)).
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 137 — Converging a machine cut off its own guests, and nothing said so
|
||||
|
||||
## What was observed
|
||||
|
||||
A workstation was flipped from adopted to converged, so the mesh's derived filter replaced what was
|
||||
there. The flip reported success, the machine reported that it had applied its declaration, and every
|
||||
surface of the mesh read green.
|
||||
|
||||
A container on one of that machine's networks could no longer reach anything:
|
||||
|
||||
```
|
||||
192.168.64.2/20
|
||||
OUTBOUND BLOCKED
|
||||
```
|
||||
|
||||
Five of the machine's container networks were affected, and every network its test beds create. The
|
||||
reason is in the filter's forward chain, which denies by default and then allows two ranges:
|
||||
|
||||
```
|
||||
ip saddr 172.16.0.0/12 accept # the container runtime's bridge networks
|
||||
ip saddr 192.168.128.0/17 accept # the networks its compose files are given
|
||||
```
|
||||
|
||||
Those two are constants in the controller. The machine's guests were allocated from neither: its
|
||||
compose networks from other parts of `192.168/16`, and each test bed a fresh `10.x/24`. So the rules
|
||||
were correct for a machine whose runtime was left at its defaults, and a guess on this one.
|
||||
|
||||
Two further things were closed by the same flip, and for the same reason nobody saw them: a guest asks
|
||||
its host for an address over DHCP and for names over DNS, both of which arrive at the input chain,
|
||||
where no module had declared them.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The preview could not have warned.** It lists what the machine reported as *listening*, and says so
|
||||
honestly: it ends with a line that traffic the machine routes is "not previewed". What it did not say
|
||||
is that routing was about to be denied by default, or which ranges would survive. An operator reading
|
||||
a 350-line preview approves what it shows.
|
||||
|
||||
**It is the second time today that a constant stood in for something the mesh cannot know.** The
|
||||
intrusion-prevention module named a firewall front-end two machines do not have
|
||||
([issue 136](../136-a-module-may-name-a-program-the-machine-does-not-have/00-report.md)), and the
|
||||
filter names the address ranges one runtime happens to use. Both were true where they were written and
|
||||
silently false elsewhere.
|
||||
|
||||
**And the code already knew.** The comment above those two lines says a machine configured otherwise
|
||||
"needs this to say so — which is a thing the mesh cannot derive and a reason this list is named here
|
||||
rather than computed". The gap was documented at the point where it was introduced, and the way to say
|
||||
it was never built. A comment naming a missing mechanism is a rule that is not enforced.
|
||||
|
||||
## What was done
|
||||
|
||||
A machine says which networks it routes; the filter forwards them and admits their guests' address and
|
||||
name service. Added to the runtime's defaults rather than replacing them, so a machine that names one
|
||||
range keeps the others. Node-level, because the machine routes them and the module that loads the
|
||||
filter may be replaced. The converge preview now says what a machine routes, and what it will keep
|
||||
forwarding if it says nothing.
|
||||
|
||||
## What is still true
|
||||
|
||||
**The flip is still the moment a machine's unmanaged services close.** That is what converging means
|
||||
and the preview names each one. This issue is not about the ports that were meant to close; it is about
|
||||
the ones nothing could name.
|
||||
|
||||
**Egress is still not previewed per network.** The preview says which ranges will be forwarded, not
|
||||
which of the machine's guests sit inside them. Deriving that would need the machine to report its
|
||||
bridges, and a bed's bridge does not exist until the bed runs.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
status: open
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-controller internal/catalogue, mesh-catalog]
|
||||
fixed-by:
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 138 — Two modules claim one seat and are not interchangeable, and nothing says so
|
||||
|
||||
## What was observed
|
||||
|
||||
Three modules claim the node-scoped uplink seat: one for each network manager a machine here might
|
||||
run. [ADR 0117](../../02-DECISIONS/0117-a-machines-uplink-is-a-seat.md) gives each of them the same
|
||||
job — ask the manager the machine already runs to leave the resolver file alone and to leave the mesh's
|
||||
interface alone — and deliberately keeps the machine's own links out of the mesh's hands.
|
||||
|
||||
A seat means one holder and an interchangeable holder. These are interchangeable in what they *ask*
|
||||
and not in what they *do*:
|
||||
|
||||
- None installs, enables, starts or stops the manager. That is on purpose: stopping it takes every
|
||||
link down, including the mesh's own way in.
|
||||
- None carries an address, a route or a wireless credential, for the same reason.
|
||||
- **Nothing checks that the module holding the seat names the manager the machine is actually
|
||||
running.** Assigning the systemd-networkd holder to a machine running NetworkManager writes a file
|
||||
for a daemon that is inactive and disabled, the seat reports held, and the two things the seat
|
||||
exists to arrange are arranged for nobody. NetworkManager goes back to rewriting the resolver file
|
||||
on every lease, which is the failure the module's own comment describes.
|
||||
|
||||
The machine reports which service manager and which units are active, so the fact needed to catch this
|
||||
is already in the report the mesh holds.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**A seat is the mesh's promise that a role is filled.** If the holder can be a module for software
|
||||
that is not running, the seat says a role is filled while nothing fills it — and the surface that
|
||||
would tell an operator says "held".
|
||||
|
||||
**It is the same shape as two faults found the same day.** A module named a firewall front-end the
|
||||
machine does not have ([issue 136](../136-a-module-may-name-a-program-the-machine-does-not-have/00-report.md)),
|
||||
and the filter named address ranges one runtime happens to use
|
||||
([issue 137](../137-converging-a-machine-cut-off-its-own-guests/00-report.md)). Each is a claim about
|
||||
the machine that nothing on the machine checks.
|
||||
|
||||
**And it decides whether the seat is worth having.** Either the holder must match what the machine
|
||||
runs, which is a condition the mesh can check from the report it already has, or the holders must be
|
||||
able to switch the manager, which ADR 0117 refuses for a reason that has not changed.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should a seat's conditions of holding include a capability the machine reports, so a holder naming
|
||||
absent or inactive software is refused rather than recorded?
|
||||
- Is "the uplink" one seat at all, if its holders are three dialects of the same two requests? The
|
||||
alternative is one module that speaks whichever dialect the machine needs, chosen from the report.
|
||||
- What should happen on a machine that switches manager afterwards? The seat would then be held by the
|
||||
wrong module, and the machine is the only place that knows.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
status: open
|
||||
opened: 2026-09-28
|
||||
located-in: [mesh-controller internal/catalogue]
|
||||
fixed-by:
|
||||
amended-design:
|
||||
---
|
||||
|
||||
# 139 — An internal route name resolves to the consumer's node, not the one that serves it
|
||||
|
||||
## What was observed
|
||||
|
||||
A module that requires a route is given two names: a public one composed under the serving node's
|
||||
domain, and an internal one composed under the consumer's own machine — `<label>.<node>.internal`.
|
||||
|
||||
The two are published differently:
|
||||
|
||||
- The **public** name is written into every machine's hosts file at the address of the node whose
|
||||
proxy answers it. The mesh computes that deliberately, so any container resolving a routed name
|
||||
reaches the proxy.
|
||||
- The **internal** name is resolved by the machine's own resolver, which answers every name under
|
||||
`<node>.internal` with that node's address — the consumer's, because the name was composed from it.
|
||||
|
||||
Where the proxy runs beside the consumer these are the same machine, which is every case on this mesh
|
||||
today, and both names work. Measured on 2026-09-28: the internal name of a service on the control node
|
||||
answers with a certificate from the mesh's internal authority, and the public name with one from the
|
||||
public authority.
|
||||
|
||||
Where the proxy is on another machine they disagree. The internal name sends the client to a machine
|
||||
that runs no proxy and has nothing listening on the port, while the public name sends it to the one
|
||||
that does.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**It is latent exactly where the mesh is heading.** `route` is provided mesh-wide precisely so a
|
||||
module can be routed by a proxy on another machine. The first module assigned that way gets an
|
||||
internal name that does not work, and the public one that does — with no error anywhere, because both
|
||||
names resolve.
|
||||
|
||||
**A per-machine name is what an operator will reach for.** `<service>.<machine>.internal` reads like a
|
||||
promise that the service on that machine is reachable there, and the wildcard makes every such name
|
||||
resolve whether or not anything answers.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should the internal name be composed under the serving node, like the public one, or should it stay
|
||||
the consumer's and be published at the serving node's address like the public name is?
|
||||
- Is a per-node route holder the real answer — a proxy on every machine that serves its own names —
|
||||
and if so, is `route` still one mesh-wide provision or a node-scoped seat with a mesh-wide fallback?
|
||||
- What certifies the name in either case? The certificate is obtained by whoever terminates TLS, and
|
||||
that is the question above in another form.
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
status: located
|
||||
opened: 2026-09-28
|
||||
located-in:
|
||||
- mesh-controller internal/catalogue/manifest.go
|
||||
- mesh-controller internal/catalogue/filtering.go
|
||||
- mesh-controller internal/catalogue/declaration.go
|
||||
- mesh-controller examples/route-proxy
|
||||
- mesh-catalog (every routed module manifest)
|
||||
fixed-by:
|
||||
amended-design: 03-DESIGN/01-to-be/08-connectivity.md
|
||||
---
|
||||
|
||||
# 140 — An endpoint's reach is not declared, so three mechanisms each decide it separately
|
||||
|
||||
## What was observed
|
||||
|
||||
Preparing to converge the mesh's control-node — the last machine still running the firewall it
|
||||
had before the mesh — the question came up for one module: the forge serves git over ssh, and that
|
||||
port must stay reachable from outside the private network. Where is that said?
|
||||
|
||||
The manifest declares the port with a source of `mesh`, so the derived filter would close it to
|
||||
everything but the private network. Looking for the place an assignment says otherwise, there are
|
||||
two per-node settings keys: one that gives a module's declared port a machine port, and one that
|
||||
overrides a declared port's source. The second has exactly one caller — the function that builds
|
||||
the node's filter rules. Nothing else in the control plane reads it.
|
||||
|
||||
A module's routed endpoint is declared somewhere else entirely: a route contribution naming a label
|
||||
and a port. It says nothing about reach. The proxy composes a **public** name and an **internal**
|
||||
name for every route it is given, and obtains a certificate for each from a different authority.
|
||||
Measured on that machine the same day: an identity provider's public name signed by the public
|
||||
authority for 90 days, its internal name signed by the mesh's own intermediate for 24 hours and
|
||||
renewed daily. Both names exist, and both certificates, because the proxy makes every name it can.
|
||||
No assignment asked for either.
|
||||
|
||||
So the forge's ssh endpoint has a firewall source and nothing else — no name, no certificate, and no
|
||||
way to say it should be public other than a key the filter alone reads. And the forge's web endpoint
|
||||
has two names and two certificates that nobody requested.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**Reach is stated twice, in two vocabularies, in two places that cannot disagree out loud.** A port
|
||||
may be exposed to anywhere while the module contributes no public route; a public route may be served
|
||||
for a module whose own listen is private. Nothing reconciles the pair or refuses it. Each mechanism
|
||||
is separately defensible and the combination is unstated.
|
||||
|
||||
**The vocabulary belongs to the filter, not to reachability.** *Public, internal, or both* cannot be
|
||||
expressed. A source of `anywhere` is one rule on one chain; it says nothing about which names should
|
||||
exist or which authority should sign them. So "this endpoint must not be public" has no way to be
|
||||
written, and is therefore enforced by nothing — while a public certificate for that very name is
|
||||
obtained automatically.
|
||||
|
||||
**An endpoint is not a thing in the model.** A module has ports, and separately it has routes.
|
||||
Nothing binds a port to a name to a certificate, which is why three mechanisms each decide reach on
|
||||
their own and none of them is wrong. This is
|
||||
[ADR 0045](../../02-DECISIONS/0045-a-machine-firewall-is-the-sum-of-what-it-listens-on.md)'s fault
|
||||
one level up: that record closed "a declaration that reads as a restriction and restricts nothing"
|
||||
for the packet filter. Here the declaration is absent altogether and the mechanisms guess.
|
||||
|
||||
**It blocks the certificate work.** The open question recorded for certificates — a name that must
|
||||
not be public needs either DNS-01 or the internal authority only — cannot be answered while no
|
||||
assignment states whether a name should be public. Neither can expiry reporting, revocation, or what
|
||||
happens to a name when a machine leaves: all of them need to know which names were *meant*.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should an assignment name each of a module's endpoints, bind it to a node-level port, and state
|
||||
whether it is reachable publicly, internally or both — with the filter, the proxy's names and the
|
||||
certificate authority all derived from that one statement?
|
||||
- What is an endpoint that is neither routed nor certified? Git over ssh is public reach with no name
|
||||
and no certificate; the model has to hold that without inventing one.
|
||||
- Are the two existing settings keys the same statement, half-built? If so, is this a new declaration
|
||||
or the completion of theirs?
|
||||
- Does an internal-only endpoint get a certificate at all, and from which authority — and does that
|
||||
settle [issue 129](../129-nothing-makes-a-machine-trust-the-meshs-authority/00-report.md), where nothing
|
||||
installs the mesh's own root?
|
||||
- Does declaring reach per assignment also settle
|
||||
[issue 139](../139-an-internal-route-name-resolves-to-the-consumers-node/00-report.md), where an
|
||||
internal name resolves to the consumer's machine instead of the one serving the endpoint?
|
||||
@@ -0,0 +1,88 @@
|
||||
---
|
||||
status: located
|
||||
opened: 2026-09-28
|
||||
located-in:
|
||||
- mesh-controller internal/catalogue/filtering.go
|
||||
- mesh-host internal/apply
|
||||
fixed-by:
|
||||
amended-design: 03-DESIGN/01-to-be/08-connectivity.md
|
||||
---
|
||||
|
||||
# 141 — The forward chain does not follow the modules, though the modules declare their networks
|
||||
|
||||
## What was observed
|
||||
|
||||
[ADR 0137](../../02-DECISIONS/0137-a-machine-says-which-networks-it-routes.md), decided the same
|
||||
week, gave a machine a way to say which networks it routes for its guests, because the derived
|
||||
filter's forward chain had until then allowed two ranges named as constants in the control plane's
|
||||
own source — the container runtime's default bridge pool, and half of the pool its compose files
|
||||
are given.
|
||||
|
||||
Checking the last machine still to be converged, the same fault was found to be live there, and the
|
||||
declaration needed to work around it turned out to be wrong in kind.
|
||||
|
||||
That machine hosts twenty-one container networks. Nine fall inside the runtime's bridge pool and
|
||||
are forwarded. Twelve sit in the other private range, and **six of those fall below the lower bound
|
||||
of the constant**, so the flip would have cut their guests off exactly as it did on the workstation
|
||||
that produced 0137.
|
||||
|
||||
Naming a range to cover the six was the obvious move, and is what 0137 provides for. But of those
|
||||
six networks, **four are networks the mesh's own modules declare** — they appear as network
|
||||
resources in the node's plan, created by the host because a module asked for them — and **two are
|
||||
leftovers of the predecessor**, compose networks of services the mesh does not run. A range wide
|
||||
enough to keep the four would have forwarded the two as well: a firewall widened by hand to protect
|
||||
networks that should not exist.
|
||||
|
||||
The mesh already knows which of the twenty-one are its own. It made them.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The node's configuration is supposed to follow the modules assigned to it.** That is the mesh's
|
||||
founding shape — the machine runs modules, and its files, its filter and its accounts are composed
|
||||
from what runs there. The forward chain is the one derived thing that does not: it consults two
|
||||
constants and, since 0137, a list a person types. A module added tomorrow brings a network the filter
|
||||
will not forward; a module deprecated leaves a range in the list that outlives it.
|
||||
|
||||
**A typed range cannot distinguish the mesh's networks from what was left behind.** It is stated in
|
||||
addresses, and addresses are what the runtime allocates, so the only honest declaration is one wide
|
||||
enough to include whatever else the runtime has handed out. The derivation is narrower than anything
|
||||
a person can safely write, because it names networks rather than ranges.
|
||||
|
||||
**0137 rejected deriving this, and was right about what it rejected.** It considered deriving the
|
||||
list from *what the machine reports* and refused, on two grounds: a test bed creates its bridge
|
||||
between one declaration and the next, and a filter that follows whatever appeared on the machine is a
|
||||
firewall that widens itself. Deriving from the **declaration** is neither. The set is known before
|
||||
the network exists, because a module declared it; and it cannot widen itself, because only a network
|
||||
some module asked for is ever forwarded. What remains genuinely for a machine to say is guests no
|
||||
module declares — a test bed's pool — which is a much smaller residue than the list as it stands.
|
||||
|
||||
**The gap is invisible in the one place that should show it.** The converge preview lists what
|
||||
*listens*, and routing is not a listener. It says in one line what the machine routes, and a reader
|
||||
has to know the runtime's allocations to tell whether that line is sufficient. On the machine
|
||||
measured here it read as though nothing needed saying.
|
||||
|
||||
## What was decided
|
||||
|
||||
*2026-09-28, later the same day.* The answer is not a better list. The question in the first open
|
||||
item below — should the chain be derived from the networks the modules declare — was answered *no*,
|
||||
after a converged machine's rendered rules were read: the chain blocks everything passing through the
|
||||
machine and then allows its own guests back by listing their addresses. Every route to a correct list
|
||||
fails, because the mesh has no position on a container reaching outward in the first place. The filter
|
||||
now constrains what arrives from **outside** the machine and says nothing about what did not, and a
|
||||
machine says which of its links face outside — one reported fact instead of a list. See
|
||||
[ADR 0140](../../02-DECISIONS/0140-the-filter-constrains-what-arrives-from-outside.md), which
|
||||
supersedes both 0137 and the first attempt at answering this.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should the forward chain be derived from the network resources the node's modules declare, with the
|
||||
host resolving each declared network to its address the way it already resolves a container by name?
|
||||
The controller cannot render the address itself: a module's network resource carries a name, and the
|
||||
runtime allocates the subnet at creation.
|
||||
- What remains of `node networks` once that exists — only guests no module declares, such as a test
|
||||
bed's pool? And should it then be named for that, rather than for all routing?
|
||||
- The runtime's own default bridge, which containers attach to when no module network is named, is
|
||||
not a module's network. Is it derived from the machine, declared by the module that owns the
|
||||
runtime, or left as the one constant?
|
||||
- Should the preview say which of a machine's networks are the mesh's and which are not, so a range
|
||||
that exists to protect a leftover is visible as such?
|
||||
@@ -0,0 +1,76 @@
|
||||
---
|
||||
status: located
|
||||
opened: 2026-09-29
|
||||
located-in:
|
||||
- mesh-host internal/upgrade
|
||||
- mesh-host cmd/mesh-host
|
||||
- mesh-controller (no build source for the host; no resource delivers it)
|
||||
fixed-by:
|
||||
amended-design: 03-DESIGN/01-to-be/05-the-node-host.md
|
||||
---
|
||||
|
||||
# 142 — The host is the one thing the mesh does not deliver
|
||||
|
||||
## What was observed
|
||||
|
||||
A change to the host was merged and could not reach any machine without a person copying a file.
|
||||
|
||||
Checked on the mesh of four machines, 2026-09-29:
|
||||
|
||||
- **The host is not a build target.** Asked what had been built for it, the control plane answered
|
||||
`nothing has been built for mesh-host`. A merge on the forge builds every changed module and the
|
||||
control plane itself, because the control plane is a module. The host is not one, and nothing
|
||||
builds it.
|
||||
- **No declaration delivers it.** No resource kind names an executable to place on a machine, and
|
||||
nothing on a machine fetches one.
|
||||
- **The half that recovers from a bad host exists and is unused.** `internal/upgrade` can report that
|
||||
the executable this process started from has been replaced on disk, and records which version last
|
||||
completed a reconcile so a shell script can roll back a host that will not start. The launcher reads
|
||||
that record and rolls back. But `Replaced()` is called by nothing except its own tests — the
|
||||
recovery is wired and the delivery was never built.
|
||||
- **Every machine runs a byte-identical binary, stamped by hand.** All four carry the same size and
|
||||
the same timestamp, from the last time somebody built it on a workstation and copied it out. No
|
||||
package owns the file.
|
||||
|
||||
## Why it matters beyond this instance
|
||||
|
||||
**The component that implements updating is the one thing not updated.** The mesh's stated shape is
|
||||
that a push produces the right builds and they reach the machines running them with nobody asking. It
|
||||
is true of every module and of the control plane. It is false for the host, which is what applies all
|
||||
of them.
|
||||
|
||||
**It is a bootstrap problem being answered by a person.** The host cannot be an ordinary module
|
||||
because the host is what applies modules; a module that replaces the thing applying it has to survive
|
||||
its own replacement. That is a real difficulty, and the work already done — noticing that the
|
||||
executable changed, recording a known-good version, a launcher that rolls back — is the hard half of
|
||||
solving it. What is missing is the easy half, and its absence makes the hard half dead code.
|
||||
|
||||
**A hand-copied binary has no record anywhere.** Nothing says which version a machine runs, so
|
||||
nothing can say a machine is behind, and the mesh's own account of itself — every machine current with
|
||||
its source — cannot include the host. Four machines agreeing today is luck, not a property.
|
||||
|
||||
**And it silently gates any change that starts in the host.** A change that needs the host to report
|
||||
something new cannot be rolled out by merging it: the control plane must wait for a person, and until
|
||||
then it either refuses what depends on the new report or renders something wrong. That cost is paid by
|
||||
every future change of this shape, and it was paid today.
|
||||
|
||||
## Open questions
|
||||
|
||||
- How is the host delivered without being applied by itself? A candidate shape: the host is built like
|
||||
anything else, published as an artifact, and the *running* host fetches and stages the next one, then
|
||||
stands aside — which is what `Replaced()` was written for and what the launcher's rollback already
|
||||
covers.
|
||||
- **Should this ride the bus, rather than becoming a mechanism of its own?** Everything else that
|
||||
reaches a machine already does: a declaration is sent over it, a report comes back over it, and a
|
||||
build announces what it produced on it, which is how a module's new version reaches the machines
|
||||
running it. A host build announcing itself the same way, consumed by the host already running,
|
||||
would make this the existing mechanism pointed at one more artifact rather than a second way of
|
||||
delivering things. It would also give the machine somewhere to say which host it is running, on the
|
||||
report it already sends.
|
||||
- What records which version of the host a machine runs, so "behind" is answerable? Nothing does now.
|
||||
- Does the host's version belong in its report, beside the other facts a machine states about itself?
|
||||
- Who decides when a machine takes a new host — the mesh, on a build, or an operator per machine as
|
||||
with converging? The rollback path means a bad host costs a reconcile rather than a machine, which
|
||||
argues for the former.
|
||||
- Does the same gap apply to the launcher and the units beside the binary, which are also files no
|
||||
declaration names?
|
||||
Reference in New Issue
Block a user