Archived
Compare commits
1
Commits
main
..
130d20114b
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
130d20114b |
+1
-8
@@ -2,11 +2,4 @@
|
|||||||
|
|
||||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||||
# built artifacts belong in build/, which is already ignored.
|
# built artifacts belong in build/, which is already ignored.
|
||||||
/mesh-controller
|
/mesh-builder
|
||||||
/postgres-provisioner
|
|
||||||
/redis-provisioner
|
|
||||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
|
||||||
/cmd/mesh-controller/mesh-controller
|
|
||||||
|
|
||||||
# No repository commits its dependencies (issue 1305).
|
|
||||||
vendor/
|
|
||||||
|
|||||||
+41
@@ -0,0 +1,41 @@
|
|||||||
|
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||||
|
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||||
|
# `make image` broke once before (issue 146).
|
||||||
|
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||||
|
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||||
|
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||||
|
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||||
|
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||||
|
#
|
||||||
|
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||||
|
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||||
|
# holds the program and nothing else — no shell, no package manager, no libc, nothing with a CVE
|
||||||
|
# feed of its own. What a person has to audit before trusting a first node is one binary.
|
||||||
|
#
|
||||||
|
# There are no CA certificates in here on purpose. Nothing it does today makes an outbound TLS
|
||||||
|
# connection to a public name: it reaches PostgreSQL on the machine it was raised on, and the
|
||||||
|
# broker is verified against a fingerprint pinned in a token rather than against a public root
|
||||||
|
# (novox/hq ADR 0004). Adding them "just in case" would put a trust store in the one image whose
|
||||||
|
# whole argument is that it contains nothing to reason about.
|
||||||
|
|
||||||
|
FROM ${GO_BASE} AS build
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Dependencies first, so a change to the source does not refetch them.
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
ARG VERSION=development
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath \
|
||||||
|
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||||
|
-o /mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
COPY --from=build /mesh-controller /mesh-controller
|
||||||
|
|
||||||
|
# Numeric because there is no /etc/passwd to look a name up in. Nothing here needs to be root:
|
||||||
|
# it opens outbound connections and writes nothing to its own filesystem.
|
||||||
|
USER 65534:65534
|
||||||
|
|
||||||
|
ENTRYPOINT ["/mesh-controller"]
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
# novox/hq ADR 0006 — the control plane, in Go.
|
||||||
|
#
|
||||||
|
# The image the bundle pins holds the program and nothing else, so the build is static and the
|
||||||
|
# container is built FROM scratch. That is not a size optimisation: this image is fetched by
|
||||||
|
# digest and run on a machine where no mesh exists to check anything, and everything in it is
|
||||||
|
# something a person would have to audit.
|
||||||
|
|
||||||
|
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo development)
|
||||||
|
LDFLAGS := -s -w -X main.version=$(VERSION)
|
||||||
|
|
||||||
|
# Where `make check` raises PostgreSQL. A high port and a throwaway container: nothing here
|
||||||
|
# touches a database anybody else is using. Override PG_PORT if this one is taken -- the first
|
||||||
|
# port chosen was already serving something that had been up for six days.
|
||||||
|
PG_PORT ?= 55532
|
||||||
|
PG_CONTAINER ?= mesh-controller-check
|
||||||
|
PG_IMAGE ?= postgres:17-alpine
|
||||||
|
export MESH_TEST_POSTGRES ?= postgres://postgres:check@127.0.0.1:$(PG_PORT)/postgres?sslmode=disable
|
||||||
|
|
||||||
|
.PHONY: build image check test vet fmt postgres postgres-stop clean
|
||||||
|
|
||||||
|
build:
|
||||||
|
CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-controller ./cmd/mesh-controller
|
||||||
|
|
||||||
|
# Tagged 'development' as well as by version, because the lab places images by name and a
|
||||||
|
# scenario naming a version would have to be edited on every build. The version tag is what a
|
||||||
|
# real bundle pins.
|
||||||
|
IMAGE ?= mesh-controller:$(VERSION)
|
||||||
|
DEV_TAG ?= mesh-controller:development
|
||||||
|
|
||||||
|
# The Go base the image is built on.
|
||||||
|
#
|
||||||
|
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||||
|
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||||
|
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||||
|
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||||
|
# what it cost).
|
||||||
|
#
|
||||||
|
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||||
|
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||||
|
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||||
|
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||||
|
|
||||||
|
image:
|
||||||
|
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The builder ships as an image too, because it is a module the mesh assigns rather than a program
|
||||||
|
# somebody starts on a machine by hand.
|
||||||
|
BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
||||||
|
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||||
|
|
||||||
|
builder-image:
|
||||||
|
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The provisioner ships as an image too, because it is the thing that makes a sealed credential
|
||||||
|
# true on a machine -- and the mesh cannot, having discarded the plaintext.
|
||||||
|
PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
||||||
|
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
|
||||||
|
|
||||||
|
provisioner-image:
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
|
||||||
|
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The object store's provisioner, for the same reason: a bucket and a policy are not files, and
|
||||||
|
# the mesh cannot make them -- it discarded the credential it would have to use.
|
||||||
|
OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
|
||||||
|
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
|
||||||
|
|
||||||
|
objectstore-image:
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
|
||||||
|
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The cache's provisioner, for the same reason as the database's: an ACL user is not a file,
|
||||||
|
# and the mesh cannot make one -- it discarded the credential it would have to use.
|
||||||
|
REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
|
||||||
|
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
|
||||||
|
|
||||||
|
redis-provisioner-image:
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
|
||||||
|
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The proxy that turns a route grant into traffic reaching a workload.
|
||||||
|
PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||||
|
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||||
|
|
||||||
|
proxy-image:
|
||||||
|
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||||
|
@echo
|
||||||
|
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||||
|
|
||||||
|
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||||
|
# including when the tests fail, which is why the teardown is not conditional.
|
||||||
|
#
|
||||||
|
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||||
|
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||||
|
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||||
|
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||||
|
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||||
|
check: fmt vet postgres
|
||||||
|
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||||
|
|
||||||
|
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||||
|
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||||
|
test:
|
||||||
|
go test -p 1 ./...
|
||||||
|
|
||||||
|
vet:
|
||||||
|
go vet ./...
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||||
|
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||||
|
|
||||||
|
postgres:
|
||||||
|
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||||
|
@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
|
||||||
|
-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
|
||||||
|
@printf 'waiting for postgres'
|
||||||
|
@for i in $$(seq 1 60) ; do \
|
||||||
|
if docker exec $(PG_CONTAINER) pg_isready -U postgres >/dev/null 2>&1 ; then \
|
||||||
|
echo ' — ready' ; exit 0 ; fi ; \
|
||||||
|
printf '.' ; sleep 1 ; \
|
||||||
|
done ; \
|
||||||
|
echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
|
||||||
|
|
||||||
|
postgres-stop:
|
||||||
|
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
clean:
|
||||||
|
rm -rf build/
|
||||||
@@ -1,7 +1,218 @@
|
|||||||
# mesh-controller
|
# mesh-controller
|
||||||
|
|
||||||
The controller ended under issue 1126. Its jobs moved to the parts that ADR 0590 and ADR 0593 name: the indexer, the
|
**Tier 2 of Novox Mesh — the control plane.** Everything that needs to know about more than one
|
||||||
assigner, the provisioner, the grants part and the node-engine. Its last module, route-proxy, moved to mesh-networking.
|
node.
|
||||||
|
|
||||||
This repository holds no code and is archived once the delivery of the group that emptied it (release sdk1) is done.
|
That is the whole test, and it draws the line the host cannot: the host applies and does not
|
||||||
Its history stays here for reading.
|
decide, *because deciding needs knowledge one machine does not have*. Which nodes should run the
|
||||||
|
store, which peers belong in an overlay, whether a node has been unreachable for a week — nobody
|
||||||
|
on a single machine can answer any of them.
|
||||||
|
|
||||||
|
The reasoning lives in [novox/hq](https://git.novox.be/novox/hq); this repository carries no
|
||||||
|
argument that is not settled there.
|
||||||
|
|
||||||
|
## What it is not
|
||||||
|
|
||||||
|
- **Not the thing that changes machines.** It decides; the host applies. It never reaches into a
|
||||||
|
node except through the host, over the link, in a bounded vocabulary.
|
||||||
|
- **Not a database.** There is no mesh database. Each context owns its own store and nothing
|
||||||
|
outside a context touches it — including nodes, which hold no credential to any of them.
|
||||||
|
- **Not privileged.** It has no more access to a machine than a declaration can express.
|
||||||
|
|
||||||
|
## What exists today
|
||||||
|
|
||||||
|
**One context of seven, and one of the things it will do.**
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| `inventory` | node records and enrolment tokens — **built, as far as identity** |
|
||||||
|
| `identity` | the control plane's signing key, and the keys nodes are known by — **built** |
|
||||||
|
| `config`, `connectivity`, `provisioning`, `delivery`, `observability` | not built |
|
||||||
|
| the interface every surface speaks to | not built; its shape is not decided |
|
||||||
|
|
||||||
|
```
|
||||||
|
mesh-controller migrate bring each context's schema up to date
|
||||||
|
mesh-controller node add <name> create a node record
|
||||||
|
mesh-controller node list the nodes this mesh knows about
|
||||||
|
mesh-controller token issue --node <name> a one-time right to join, for an existing record
|
||||||
|
mesh-controller token issue --new <name> create the record and issue for it
|
||||||
|
mesh-controller identity show this control plane's signing key
|
||||||
|
mesh-controller broker show where the broker is, and what to expect there
|
||||||
|
mesh-controller version what this binary is
|
||||||
|
```
|
||||||
|
|
||||||
|
`migrate` is **step 3 of the foundation bootstrap** — the step the first node cannot get past, run
|
||||||
|
against a database raised moments earlier from the bundle the host carries.
|
||||||
|
|
||||||
|
### Tokens, and what they are missing
|
||||||
|
|
||||||
|
A token is **a one-time right to join, issued for a node record** — which is where re-enrolment is
|
||||||
|
decided, since what an identity binds to is settled when the token is made rather than when it is
|
||||||
|
presented.
|
||||||
|
|
||||||
|
What is built: the secret is 256 bits from the system's random source, shown once, and **stored
|
||||||
|
only as a hash**, so a copy of this database is not a set of working credentials. It is usable
|
||||||
|
exactly once and only before it expires, and both are read from the row rather than from a status
|
||||||
|
something would have had to write. Issuing again for the same node invalidates the outstanding
|
||||||
|
one — two live tokens are two machines able to join as the same node.
|
||||||
|
|
||||||
|
Redemption is a single statement that both finds a live token and spends it, so eight concurrent
|
||||||
|
attempts on one secret produce exactly one winner. There is a test that runs them.
|
||||||
|
|
||||||
|
**A token now carries three of its four parts**, and is one line of base64 a person can copy. The
|
||||||
|
signing key is real: an Ed25519 key this control plane generates once and keeps, whose public half
|
||||||
|
travels in every token. A node believes a declaration because it carries a signature that key made
|
||||||
|
— and pinning only the broker would not do, because it would make the control plane's authority
|
||||||
|
transitive, so a compromised broker could forge declarations, and since the host applies whatever
|
||||||
|
the link delivers that is the whole machine.
|
||||||
|
|
||||||
|
**All four parts are built.** Given `MESH_BROKER_ADDRESS` and `MESH_BROKER_CERTIFICATE`, a token
|
||||||
|
carries everything ADR 0004 requires. Without them it carries two, and `token issue` prints it
|
||||||
|
**and names what is missing** rather than producing something that looks complete and cannot be
|
||||||
|
used.
|
||||||
|
|
||||||
|
**The fingerprint is derived from the certificate, never configured.** A configured one can drift
|
||||||
|
from the certificate it describes, and a drifted pin is worse than none: every node issued a token
|
||||||
|
during the drift refuses to connect, and the failure looks like an attack. It is computed over the
|
||||||
|
DER bytes — what a client actually sees on the wire — so the same certificate re-wrapped with
|
||||||
|
different line endings still produces the same pin.
|
||||||
|
|
||||||
|
### Two contexts, and the rule between them is real
|
||||||
|
|
||||||
|
`identity` is the second context and it exists partly to test a claim this repository had made and
|
||||||
|
never checked: that a context reaches only its own store. It holds `MESH_STORE_IDENTITY`;
|
||||||
|
`inventory` holds `MESH_STORE_INVENTORY`; there is no setting that reaches both and no way to ask
|
||||||
|
for one. Run `migrate` with only one and it stops, naming the grant it does not have.
|
||||||
|
|
||||||
|
A token needs a node record from one and a signing key from the other. Neither reads the other's
|
||||||
|
store — the process holding both grants asks each for its part.
|
||||||
|
|
||||||
|
### How a node is known
|
||||||
|
|
||||||
|
**The node generates a keypair; the mesh records the public half.** The same principle as SSH, and
|
||||||
|
the same rule `08-connectivity` already applies to the overlay keys — which is where this was
|
||||||
|
settled all along, though it took being asked directly to notice.
|
||||||
|
|
||||||
|
Only the public half is ever stored, and that is the property worth having: **a copy of this
|
||||||
|
database grants nothing.** It is a list of who to believe, not a set of credentials, which is what
|
||||||
|
makes *compromise of a node is compromise of that node* literally true.
|
||||||
|
|
||||||
|
Exactly one key is live per node. Re-enrolment revokes the one it replaced, in the same
|
||||||
|
transaction — two live identities for one node record is the stolen-laptop case, with the replaced
|
||||||
|
machine still believed. The database enforces it as well as the code, and there is a test running
|
||||||
|
six concurrent enrolments that fails when the constraint is removed.
|
||||||
|
|
||||||
|
**Not the machine's SSH host key**, though that was the obvious economy. Host keys are regenerated
|
||||||
|
by reinstalls and image clones, which would silently un-enrol a node; their lifecycle belongs to
|
||||||
|
sshd rather than the mesh; and a partial host has no SSH daemon at all.
|
||||||
|
|
||||||
|
## Reaching a store
|
||||||
|
|
||||||
|
**A context is granted only what it exclusively owns.** No shared writes, no read-only role on
|
||||||
|
another context's store, and no connection string that reaches more than one.
|
||||||
|
|
||||||
|
That is a rule about credentials, so it is built as one. There is no mesh-wide connection setting
|
||||||
|
and no way to ask for one:
|
||||||
|
|
||||||
|
```
|
||||||
|
MESH_STORE_INVENTORY=postgres://…/inventory
|
||||||
|
```
|
||||||
|
|
||||||
|
A process granted `inventory` holds that variable and no other. Reaching another context's store
|
||||||
|
is not a matter of restraint — it has no address for it and no credential to present. And it is
|
||||||
|
how the rule is *checked*: what a context can reach is visible in the declaration that runs it,
|
||||||
|
as the list of variables it was given.
|
||||||
|
|
||||||
|
Each context's database is named after the context. There is deliberately no database named for
|
||||||
|
the mesh as a whole.
|
||||||
|
|
||||||
|
## Migrations
|
||||||
|
|
||||||
|
Numbered, embedded in the binary, applied in order, each in a transaction with the row recording
|
||||||
|
it. The applying is four lines; the rest is refusals, and the refusals are the point:
|
||||||
|
|
||||||
|
| it stops when | because |
|
||||||
|
|---|---|
|
||||||
|
| a migration that ran has since been edited | the database holds the old version, the repository holds the new one, and nothing holds the difference |
|
||||||
|
| a migration is numbered below one that already ran | usually two branches taking the same next number — applying it now runs the schema in an order nobody tested |
|
||||||
|
| two migrations share a number | order is the entire guarantee, and two files with one number have none |
|
||||||
|
| a file in the directory is not a valid migration name | a misnamed migration would otherwise never run and nothing would say so |
|
||||||
|
| a migration is empty | it records that something happened and changes nothing, which cannot be told from a mistake |
|
||||||
|
|
||||||
|
All of them stop rather than warn. At the moment any of them is true, nobody knows what the
|
||||||
|
database contains, and there is no correct guess about a schema.
|
||||||
|
|
||||||
|
Running it again does nothing. Two copies running at once take an advisory lock, so a restart
|
||||||
|
during a slow migration does not become two runners racing.
|
||||||
|
|
||||||
|
## Building
|
||||||
|
|
||||||
|
```
|
||||||
|
make build the binary
|
||||||
|
make image the container image
|
||||||
|
make check gofmt, vet, and every test against a real PostgreSQL
|
||||||
|
```
|
||||||
|
|
||||||
|
`make check` raises a throwaway PostgreSQL in a container and takes it down afterwards, including
|
||||||
|
when the tests fail. Without one the tests that need a database **skip and say so** rather than
|
||||||
|
passing quietly — `make test` is the honest subset, not the gate.
|
||||||
|
|
||||||
|
Tests run against a real database rather than a fake because what is being tested *is* the
|
||||||
|
database's behaviour: that DDL is transactional, that an advisory lock serialises, that a
|
||||||
|
checksum mismatch is caught against a record PostgreSQL actually kept. A fake would assert that
|
||||||
|
the fake behaves as expected.
|
||||||
|
|
||||||
|
Every test here has been confirmed to fail when the behaviour it defends is removed. Two did not,
|
||||||
|
when first written, and both are now commented with what they were missing.
|
||||||
|
|
||||||
|
## Checks that cross into the host's repository
|
||||||
|
|
||||||
|
Two things are agreed between this repository and `novox/mesh-host`, and each is a separate struct
|
||||||
|
on each side. A field renamed on one of them fails **silently** — the crossing succeeds and
|
||||||
|
something is simply absent — so both are checked by handing one side's real output to the other's
|
||||||
|
real parser. Neither runs by default; each skips with a reason, because a repository that fails
|
||||||
|
without its neighbour checked out is a repository nobody can build.
|
||||||
|
|
||||||
|
**What this mesh sends, read by the host that receives it:**
|
||||||
|
|
||||||
|
```
|
||||||
|
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
||||||
|
mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
|
||||||
|
```
|
||||||
|
|
||||||
|
**What a node says when it joins, read by this mesh:**
|
||||||
|
|
||||||
|
```
|
||||||
|
mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/
|
||||||
|
mesh-controller: MESH_ENROL=/tmp/enrol.json make check
|
||||||
|
```
|
||||||
|
|
||||||
|
The second does more than compare shapes: it seals something to the key that arrived and opens it
|
||||||
|
with the private half the host kept. A key that is present, correctly named and simply *wrong*
|
||||||
|
passes every check that only looks at the message.
|
||||||
|
|
||||||
|
## The image
|
||||||
|
|
||||||
|
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||||
|
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||||
|
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||||
|
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||||
|
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||||
|
mesh's own build any more — `make image` builds it.
|
||||||
|
|
||||||
|
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||||
|
manager, no libc, no CA certificates.
|
||||||
|
|
||||||
|
Not a size optimisation. The bundle a host carries pins this image by digest, and it is fetched
|
||||||
|
and run on a machine where no mesh exists to check anything and a person is expected to have read
|
||||||
|
the bundle and believed it. Everything in the image is something that person would have to audit.
|
||||||
|
|
||||||
|
## Where the reasoning lives
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| what the control plane is | novox/hq ADR 0006 |
|
||||||
|
| what it takes to run one, and why Go | novox/hq ADR 0006 |
|
||||||
|
| a context owns its store, exclusively | novox/hq ADR 0008 |
|
||||||
|
| schema changes are numbered migrations | novox/hq ADR 0013 |
|
||||||
|
| a test defends a decision | novox/hq ADR 0017 |
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
ARG ALPINE_BASE=alpine:3
|
||||||
|
ARG GO_BASE=golang:1.25-alpine
|
||||||
|
# The builder, as a module ships one.
|
||||||
|
#
|
||||||
|
# Not FROM scratch, unlike the control plane: this one runs git and a container client, so it
|
||||||
|
# needs a filesystem with them on it. That is the honest cost of a machine whose job is to build —
|
||||||
|
# and it is why building is a MODULE on a machine that has a runtime rather than something the
|
||||||
|
# control plane does (novox/hq ADR 0005).
|
||||||
|
FROM ${GO_BASE} AS build
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||||
|
|
||||||
|
FROM ${ALPINE_BASE}
|
||||||
|
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||||
|
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||||
|
# rather than running one inside itself.
|
||||||
|
RUN apk add --no-cache git docker-cli
|
||||||
|
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||||
|
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||||
@@ -0,0 +1,386 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||||
|
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||||
|
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||||
|
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||||
|
// it, pause, resume.
|
||||||
|
//
|
||||||
|
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||||
|
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||||
|
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||||
|
|
||||||
|
// holder is this machine's state as a holder of the build seat.
|
||||||
|
type holder struct {
|
||||||
|
on, seat string
|
||||||
|
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||||
|
// rather than silently taking work again.
|
||||||
|
workspace string
|
||||||
|
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||||
|
say func(link.HolderState) error
|
||||||
|
// remove runs what a kill needs outside the build: the containers left behind.
|
||||||
|
remove builder.Runner
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
paused bool
|
||||||
|
running *running
|
||||||
|
}
|
||||||
|
|
||||||
|
// running is the build this machine is doing.
|
||||||
|
type running struct {
|
||||||
|
request link.BuildRequest
|
||||||
|
step string
|
||||||
|
started time.Time
|
||||||
|
cancel context.CancelFunc
|
||||||
|
killed bool
|
||||||
|
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||||
|
// announced, and announced whether it went out.
|
||||||
|
returned bool
|
||||||
|
outcome string
|
||||||
|
announced bool
|
||||||
|
done chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedFile is where the flag lives in the workspace.
|
||||||
|
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||||
|
|
||||||
|
// newHolder reads the paused flag the workspace keeps.
|
||||||
|
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||||
|
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||||
|
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||||
|
h.paused = true
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
// Paused is asked by the taking loop before every fetch.
|
||||||
|
func (h *holder) Paused() bool {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.paused
|
||||||
|
}
|
||||||
|
|
||||||
|
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||||
|
// it is and what it would be after a restart never differ.
|
||||||
|
func (h *holder) setPaused(paused bool) error {
|
||||||
|
path := pausedFile(h.workspace)
|
||||||
|
if paused {
|
||||||
|
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||||
|
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||||
|
}
|
||||||
|
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.paused = paused
|
||||||
|
h.mu.Unlock()
|
||||||
|
h.announce()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||||
|
// controller reading an older state is a plan read as late rather than a build lost.
|
||||||
|
func (h *holder) announce() {
|
||||||
|
if h.say == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||||
|
// a pause outlives the events stream's retention.
|
||||||
|
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||||
|
h.announce()
|
||||||
|
tick := time.NewTicker(every)
|
||||||
|
defer tick.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-tick.C:
|
||||||
|
if h.Paused() {
|
||||||
|
h.announce()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// begin records the build this machine took, with the cancel that ends it.
|
||||||
|
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||||
|
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||||
|
h.mu.Lock()
|
||||||
|
h.running = r
|
||||||
|
h.mu.Unlock()
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// end clears it, and lets a kill waiting on it know it is over.
|
||||||
|
func (h *holder) end(r *running) {
|
||||||
|
h.mu.Lock()
|
||||||
|
if h.running == r {
|
||||||
|
h.running = nil
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
close(r.done)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepped records the step a build is at, for `current`.
|
||||||
|
func (h *holder) stepped(r *running, step string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r.step = step
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// currentBuild is what `current` answers.
|
||||||
|
type currentBuild struct {
|
||||||
|
On string `json:"on"`
|
||||||
|
Paused bool `json:"paused"`
|
||||||
|
Running *struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Elapsed string `json:"elapsed"`
|
||||||
|
} `json:"running,omitempty"`
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *holder) current() currentBuild {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
out := currentBuild{On: h.on, Paused: h.paused}
|
||||||
|
taking := "taking builds"
|
||||||
|
if h.paused {
|
||||||
|
taking = "paused, taking no new build"
|
||||||
|
}
|
||||||
|
if h.running == nil {
|
||||||
|
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
r := h.running
|
||||||
|
elapsed := time.Since(r.started).Round(time.Second)
|
||||||
|
out.Running = &struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Step string `json:"step,omitempty"`
|
||||||
|
Started string `json:"started"`
|
||||||
|
Elapsed string `json:"elapsed"`
|
||||||
|
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||||
|
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||||
|
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||||
|
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||||
|
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||||
|
// (killAnswer in the controller's queue.go, 75s).
|
||||||
|
var (
|
||||||
|
killRemoves = 15 * time.Second
|
||||||
|
killWaits = 20 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||||
|
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||||
|
// their label, once at once and again after the build has ended, so one created while it was being
|
||||||
|
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||||
|
// ask so it is not redelivered; the answer says whether that happened.
|
||||||
|
func (h *holder) kill(id string) (string, error) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r := h.running
|
||||||
|
if r == nil || r.request.ID != id {
|
||||||
|
doing := "nothing"
|
||||||
|
if r != nil {
|
||||||
|
doing = r.request.ID
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||||
|
}
|
||||||
|
if r.returned {
|
||||||
|
h.mu.Unlock()
|
||||||
|
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||||
|
}
|
||||||
|
r.killed = true
|
||||||
|
cancel, done := r.cancel, r.done
|
||||||
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
cancel()
|
||||||
|
removed, removeErr := h.removeContainers(id)
|
||||||
|
ended := false
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
ended = true
|
||||||
|
case <-time.After(killWaits):
|
||||||
|
}
|
||||||
|
again, againErr := h.removeContainers(id)
|
||||||
|
removed += again
|
||||||
|
if removeErr == nil {
|
||||||
|
removeErr = againErr
|
||||||
|
}
|
||||||
|
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||||
|
if removeErr != nil {
|
||||||
|
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||||
|
}
|
||||||
|
if !ended {
|
||||||
|
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||||
|
"its outcome is in", id, h.on, containers), nil
|
||||||
|
}
|
||||||
|
h.mu.Lock()
|
||||||
|
outcome, announced := r.outcome, r.announced
|
||||||
|
h.mu.Unlock()
|
||||||
|
if !announced {
|
||||||
|
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||||
|
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||||
|
containers), nil
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||||
|
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// removeContainers is one pass of removing what the build left, bounded.
|
||||||
|
func (h *holder) removeContainers(id string) (int, error) {
|
||||||
|
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||||
|
defer stop()
|
||||||
|
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||||
|
// the build killed; an error it ended with on its own is its own outcome.
|
||||||
|
func (h *holder) returned(r *running) bool {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
r.returned = true
|
||||||
|
return r.killed
|
||||||
|
}
|
||||||
|
|
||||||
|
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||||
|
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||||
|
h.mu.Lock()
|
||||||
|
r.outcome, r.announced = outcome, announced
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// handlers are the seat's verbs, as this machine answers them.
|
||||||
|
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||||
|
return map[string]link.ToolHandler{
|
||||||
|
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||||
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
var args struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||||
|
return nil, errors.New("kill needs the build's id")
|
||||||
|
}
|
||||||
|
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"said": said}, nil
|
||||||
|
},
|
||||||
|
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
if err := h.setPaused(true); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
said := h.on + " is paused: it takes no new build until resumed"
|
||||||
|
if c := h.current(); c.Running != nil {
|
||||||
|
said += "; " + c.Running.ID + " runs on and finishes"
|
||||||
|
}
|
||||||
|
return map[string]any{"said": said, "paused": true}, nil
|
||||||
|
},
|
||||||
|
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
if err := h.setPaused(false); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNothing(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "its start"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||||
|
// log it would be is the one being ended.
|
||||||
|
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
cmd.Dir = dir
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||||
|
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||||
|
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||||
|
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||||
|
//
|
||||||
|
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||||
|
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||||
|
// store, and what it serves is what this binary was built to serve.
|
||||||
|
func announcement(seat, module, node string) micro.Info {
|
||||||
|
s, _ := catalogue.SeatNamed(seat)
|
||||||
|
var endpoints []micro.EndpointInfo
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
schema, _ := json.Marshal(v.Input)
|
||||||
|
endpoints = append(endpoints, micro.EndpointInfo{
|
||||||
|
Name: seat + "__" + v.Name,
|
||||||
|
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||||
|
// The queue group the verbs are served in, as every runtime announces its own.
|
||||||
|
QueueGroup: "seat." + seat,
|
||||||
|
Metadata: map[string]string{
|
||||||
|
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||||
|
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return micro.Info{
|
||||||
|
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||||
|
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||||
|
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||||
|
Endpoints: endpoints,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||||
|
func moduleOf(user string) string {
|
||||||
|
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||||
|
return module
|
||||||
|
}
|
||||||
|
return "build-agent"
|
||||||
|
}
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||||
|
// 0219), and what it says about itself follows.
|
||||||
|
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||||
|
workspace := t.TempDir()
|
||||||
|
var said []link.HolderState
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||||
|
said = append(said, s)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if h.Paused() {
|
||||||
|
t.Fatal("a new holder starts paused")
|
||||||
|
}
|
||||||
|
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||||
|
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||||
|
}
|
||||||
|
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||||
|
if !again.Paused() {
|
||||||
|
t.Fatal("restarted, the holder forgot it was paused")
|
||||||
|
}
|
||||||
|
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||||
|
t.Fatal("resumed, the holder came back paused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||||
|
// label, and refuses an id it is not building.
|
||||||
|
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||||
|
var removed []string
|
||||||
|
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||||
|
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||||
|
if args[0] == "ps" {
|
||||||
|
return "c1\n", nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
kill := h.handlers()["kill"]
|
||||||
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||||
|
t.Fatal("killed a build while none ran")
|
||||||
|
}
|
||||||
|
|
||||||
|
building, cancel := context.WithCancel(context.Background())
|
||||||
|
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||||
|
h.stepped(r, "image")
|
||||||
|
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||||
|
t.Fatalf("current says %+v", c)
|
||||||
|
}
|
||||||
|
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "build-1") {
|
||||||
|
t.Fatalf("killed another id: %v", err)
|
||||||
|
}
|
||||||
|
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||||
|
// announces what came of it.
|
||||||
|
go func() {
|
||||||
|
<-building.Done()
|
||||||
|
if h.returned(r) {
|
||||||
|
h.said(r, link.KilledByHand, true)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
}()
|
||||||
|
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if building.Err() == nil {
|
||||||
|
t.Fatal("the build's context was not cancelled")
|
||||||
|
}
|
||||||
|
if !r.killed {
|
||||||
|
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||||
|
}
|
||||||
|
said := answer.(map[string]any)["said"].(string)
|
||||||
|
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||||
|
t.Errorf("kill said %q", said)
|
||||||
|
}
|
||||||
|
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||||
|
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||||
|
t.Errorf("removed %v", removed)
|
||||||
|
}
|
||||||
|
if c := h.current(); c.Running != nil {
|
||||||
|
t.Errorf("after the kill current says %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||||
|
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||||
|
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||||
|
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||||
|
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||||
|
}
|
||||||
|
kill := info.Endpoints[1]
|
||||||
|
md := kill.Metadata
|
||||||
|
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||||
|
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||||
|
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||||
|
t.Fatalf("kill is announced as %+v", kill)
|
||||||
|
}
|
||||||
|
body, err := json.Marshal(info)
|
||||||
|
if err != nil || len(body) > 8*1024 {
|
||||||
|
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||||
|
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||||
|
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||||
|
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||||
|
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||||
|
_, cancel := context.WithCancel(context.Background())
|
||||||
|
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||||
|
if killed := h.returned(r); killed {
|
||||||
|
t.Fatal("a build nobody killed reads as killed")
|
||||||
|
}
|
||||||
|
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||||
|
t.Fatalf("killed a build that had ended: %v", err)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
|
||||||
|
building, cancel := context.WithCancel(context.Background())
|
||||||
|
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||||
|
go func() {
|
||||||
|
<-building.Done()
|
||||||
|
if h.returned(r) {
|
||||||
|
h.said(r, link.KilledByHand, false)
|
||||||
|
}
|
||||||
|
h.end(r)
|
||||||
|
}()
|
||||||
|
said, err := h.kill("build-2")
|
||||||
|
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||||
|
t.Fatalf("kill said %q (%v)", said, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,569 @@
|
|||||||
|
// mesh-builder — the thing a build machine runs.
|
||||||
|
//
|
||||||
|
// It takes work from the mesh, turns a repository into artifacts, publishes them, and says what
|
||||||
|
// came out. It is **not** the control plane and it is **not** the host:
|
||||||
|
//
|
||||||
|
// - the control plane decides and never touches a machine. Building runs commands on one, and
|
||||||
|
// what the control plane may send a machine is bounded by the declaration language
|
||||||
|
// (novox/hq ADR 0005). "Run this build" is not in it, and widening the language so it could
|
||||||
|
// be would make the control plane able to run anything anywhere.
|
||||||
|
// - the host applies declarations and holds no opinion about what they contain. A host that
|
||||||
|
// also built things would need a container runtime and git, on every machine, to do something
|
||||||
|
// almost none of them will ever do.
|
||||||
|
//
|
||||||
|
// So it is a module: a program a machine runs because the mesh told it to, holding its own broker
|
||||||
|
// credential and nothing else. Compromise of a build machine is compromise of a build machine.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// version is set at build time.
|
||||||
|
var version = "development"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-builder: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const usage = `mesh-builder — builds modules for the mesh
|
||||||
|
|
||||||
|
It consumes build requests and answers with what it made. Nothing is listened on and nothing
|
||||||
|
is dialled except the broker.
|
||||||
|
|
||||||
|
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
|
||||||
|
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
|
||||||
|
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||||
|
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
|
||||||
|
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
|
||||||
|
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
|
||||||
|
MESH_NPM_TOKEN the token for it, likewise
|
||||||
|
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||||
|
MESH_WORKSPACE where to clone and build (default: a temporary directory)
|
||||||
|
|
||||||
|
It also builds one module and stops, which is how a mesh is raised — before there is a
|
||||||
|
broker to take work from or a registry to publish into:
|
||||||
|
|
||||||
|
mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]
|
||||||
|
|
||||||
|
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||||
|
digest of their own configuration. The result is printed as JSON.
|
||||||
|
`
|
||||||
|
|
||||||
|
func run() error {
|
||||||
|
if len(os.Args) > 1 {
|
||||||
|
switch os.Args[1] {
|
||||||
|
case "version":
|
||||||
|
fmt.Println(version)
|
||||||
|
return nil
|
||||||
|
case "build":
|
||||||
|
return buildOnce(context.Background(), os.Args[2:])
|
||||||
|
default:
|
||||||
|
fmt.Print(usage)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
credential, err := brokerFrom()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
registry, err := whereToPublish()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
workspace := os.Getenv("MESH_WORKSPACE")
|
||||||
|
if workspace == "" {
|
||||||
|
workspace = os.TempDir() + "/mesh-builder"
|
||||||
|
}
|
||||||
|
// **The mesh's name for this machine, not the container's.** A build is reported to the rest
|
||||||
|
// of the mesh, and a report whose origin reads `104cb10e105b` names something no other module
|
||||||
|
// can look up. The mesh already knows the answer and has a way to say it — `${machine:name}`
|
||||||
|
// in the environment file this module is handed — so the hostname is only what is left when
|
||||||
|
// nobody said.
|
||||||
|
on := os.Getenv("MESH_NODE")
|
||||||
|
if on == "" {
|
||||||
|
hostname, err := os.Hostname()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("this build machine has no name: nothing said MESH_NODE and the host would not say either: %w", err)
|
||||||
|
}
|
||||||
|
on = hostname
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
js, seat, err := dialFor(credential)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
|
||||||
|
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||||
|
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||||
|
// paused takes nothing.
|
||||||
|
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||||
|
body, err := json.Marshal(state)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if h.Paused() {
|
||||||
|
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||||
|
}
|
||||||
|
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||||
|
defer machine.Close()
|
||||||
|
|
||||||
|
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||||
|
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||||
|
if seat == link.TheBuildMachine {
|
||||||
|
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||||
|
log.New(os.Stderr, "", 0))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer stopServing()
|
||||||
|
// And says so, for the console to find (novox/hq ADR 0197).
|
||||||
|
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||||
|
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer stopAnnouncing()
|
||||||
|
go h.stateWhilePaused(ctx, time.Hour)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||||
|
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||||
|
|
||||||
|
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||||
|
answer(ctx, publisher, on, workspace, work, h)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||||
|
// holds.
|
||||||
|
//
|
||||||
|
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||||
|
// machine told about both would take work from one and answer on the other, and every log line would
|
||||||
|
// say it was fine.
|
||||||
|
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||||
|
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||||
|
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||||
|
// and that is enough to dial it, pinned.
|
||||||
|
if !credential.onTheNewBus() {
|
||||||
|
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||||
|
}
|
||||||
|
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", err
|
||||||
|
}
|
||||||
|
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||||
|
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||||
|
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||||
|
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||||
|
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||||
|
return js, seat, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer does one build and says what happened, whichever way it went.
|
||||||
|
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||||
|
request := work.Request()
|
||||||
|
|
||||||
|
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||||
|
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||||
|
// reaches it through the parent, and that keeps today's meaning below.
|
||||||
|
building, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
var mine *running
|
||||||
|
if h != nil {
|
||||||
|
mine = h.begin(request, cancel)
|
||||||
|
defer h.end(mine)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||||
|
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||||
|
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||||
|
// the handler said nothing until the end.
|
||||||
|
fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID)
|
||||||
|
|
||||||
|
// **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the
|
||||||
|
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||||
|
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||||
|
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||||
|
say := func(step, message string) {
|
||||||
|
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||||
|
work.Say(step, message)
|
||||||
|
if mine != nil && step != "run" && step != "output" {
|
||||||
|
h.stepped(mine, step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
builder.Said = say
|
||||||
|
defer func() { builder.Said = nil }()
|
||||||
|
if err := work.Began(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := link.BuildResult{
|
||||||
|
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||||
|
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||||
|
}
|
||||||
|
what := "building " + request.Repository
|
||||||
|
if request.Path != "" {
|
||||||
|
what += " at " + request.Path
|
||||||
|
}
|
||||||
|
if request.Ref != "" {
|
||||||
|
what += " on " + request.Ref
|
||||||
|
}
|
||||||
|
say("build", what)
|
||||||
|
|
||||||
|
npmrc, err := packagesFrom()
|
||||||
|
var built builder.Result
|
||||||
|
if err == nil {
|
||||||
|
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||||
|
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||||
|
// after a clone that then fails at npm ci.
|
||||||
|
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||||
|
// docker client (ADR 0219).
|
||||||
|
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||||
|
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||||
|
forgeFrom(), say, request.Seats)
|
||||||
|
}
|
||||||
|
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||||
|
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||||
|
killed := false
|
||||||
|
if mine != nil {
|
||||||
|
killed = h.returned(mine) && err != nil
|
||||||
|
}
|
||||||
|
if killed {
|
||||||
|
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||||
|
// error it ended with is the kill's consequence, not a fault of the source.
|
||||||
|
result.Failed = link.KilledByHand
|
||||||
|
say("failed", link.KilledByHand)
|
||||||
|
} else if err != nil {
|
||||||
|
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||||
|
// builder that is not running, and those want completely different responses.
|
||||||
|
result.Failed = err.Error()
|
||||||
|
say("failed", err.Error())
|
||||||
|
} else {
|
||||||
|
manifest, marshalErr := json.Marshal(built.Manifest)
|
||||||
|
if marshalErr != nil {
|
||||||
|
result.Failed = marshalErr.Error()
|
||||||
|
} else {
|
||||||
|
result.Commit = built.Commit
|
||||||
|
result.Manifest = manifest
|
||||||
|
for _, made := range built.Built {
|
||||||
|
result.Made = append(result.Made, link.MadeArtifact{
|
||||||
|
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
result.Against = built.Against
|
||||||
|
for _, r := range built.Read {
|
||||||
|
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
|
}
|
||||||
|
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||||
|
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||||
|
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||||
|
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||||
|
announcing := ctx
|
||||||
|
if killed {
|
||||||
|
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer stop()
|
||||||
|
announcing = fresh
|
||||||
|
}
|
||||||
|
announceErr := work.Announce(announcing, result)
|
||||||
|
if mine != nil {
|
||||||
|
h.said(mine, result.Failed, announceErr == nil)
|
||||||
|
}
|
||||||
|
if err := announceErr; err != nil {
|
||||||
|
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||||
|
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||||
|
// nothing was said at all, so another machine can try.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
|
||||||
|
// for another rather than losing it.
|
||||||
|
if err := work.Done(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||||
|
// (novox/hq ADR 0076, issue 053).
|
||||||
|
//
|
||||||
|
// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact
|
||||||
|
// store's binding does, and a sealed token file the credential the way the broker's does. The
|
||||||
|
// environment variables remain for a builder run by a person, and for the bootstrap, where there is
|
||||||
|
// no registry yet — there the result is disabled and a build that needs no mesh-published dependency
|
||||||
|
// builds anyway.
|
||||||
|
func packagesFrom() (builder.Npmrc, error) {
|
||||||
|
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
|
||||||
|
if scope == "" {
|
||||||
|
scope = "@novox"
|
||||||
|
}
|
||||||
|
|
||||||
|
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||||
|
var username string
|
||||||
|
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||||
|
}
|
||||||
|
var told struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
At string `json:"at"`
|
||||||
|
As string `json:"as"`
|
||||||
|
Serves map[string]any `json:"serves"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &told); err != nil {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||||
|
}
|
||||||
|
if told.At == "" {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf(
|
||||||
|
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||||
|
}
|
||||||
|
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||||
|
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||||
|
scheme := "https"
|
||||||
|
if s, ok := told.Serves["scheme"]; ok {
|
||||||
|
scheme = fmt.Sprintf("%v", s)
|
||||||
|
}
|
||||||
|
port, ok := told.Serves["port"]
|
||||||
|
if !ok {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||||
|
}
|
||||||
|
npmPath, ok := told.Serves["npm-path"]
|
||||||
|
if !ok {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||||
|
}
|
||||||
|
registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath)
|
||||||
|
username = told.As
|
||||||
|
}
|
||||||
|
|
||||||
|
// The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a
|
||||||
|
// generated password the provider only applies (novox/hq ADR 0048) — so with a username this is
|
||||||
|
// a password (basic auth); without one it is a bearer token a provider minted.
|
||||||
|
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||||
|
if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||||
|
}
|
||||||
|
secret = strings.TrimSpace(string(raw))
|
||||||
|
}
|
||||||
|
if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" {
|
||||||
|
username = u
|
||||||
|
}
|
||||||
|
|
||||||
|
if registry == "" && secret == "" {
|
||||||
|
return builder.Npmrc{}, nil
|
||||||
|
}
|
||||||
|
if username != "" {
|
||||||
|
return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil
|
||||||
|
}
|
||||||
|
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
||||||
|
// and sealed secret its package-registry half already reads: the forge that answers npm is the
|
||||||
|
// forge that hosts the repositories, and its provisioner applies one password to one user for
|
||||||
|
// both. Anything missing means no credential, and every clone stays anonymous — which is all a
|
||||||
|
// mesh of public repositories ever needs.
|
||||||
|
//
|
||||||
|
// The URL names the binding's own address — the machine the mesh says the forge is on — so a
|
||||||
|
// private repository is registered and built by that address, and a clone of anything else is
|
||||||
|
// never shown this credential (git's credential store matches the whole origin).
|
||||||
|
func forgeFrom() builder.GitCredential {
|
||||||
|
path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING"))
|
||||||
|
if path == "" {
|
||||||
|
return builder.GitCredential{}
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return builder.GitCredential{}
|
||||||
|
}
|
||||||
|
var told struct {
|
||||||
|
At string `json:"at"`
|
||||||
|
As string `json:"as"`
|
||||||
|
Serves map[string]any `json:"serves"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" {
|
||||||
|
return builder.GitCredential{}
|
||||||
|
}
|
||||||
|
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||||
|
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
||||||
|
if raw, err := os.ReadFile(file); err == nil {
|
||||||
|
secret = strings.TrimSpace(string(raw))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if secret == "" {
|
||||||
|
return builder.GitCredential{}
|
||||||
|
}
|
||||||
|
scheme := "https"
|
||||||
|
if s, ok := told.Serves["scheme"]; ok {
|
||||||
|
scheme = fmt.Sprintf("%v", s)
|
||||||
|
}
|
||||||
|
host := told.At
|
||||||
|
if port, ok := told.Serves["port"]; ok {
|
||||||
|
host = fmt.Sprintf("%s:%v", told.At, port)
|
||||||
|
}
|
||||||
|
made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host}
|
||||||
|
return builder.GitCredential{URL: made.String()}
|
||||||
|
}
|
||||||
|
|
||||||
|
func short(commit string) string {
|
||||||
|
if len(commit) > 8 {
|
||||||
|
return commit[:8]
|
||||||
|
}
|
||||||
|
return commit
|
||||||
|
}
|
||||||
|
|
||||||
|
// whereToPublish is the artifact store this builder uses.
|
||||||
|
//
|
||||||
|
// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the
|
||||||
|
// mesh writes it a file saying which machine answers that and on what port — the same binding any
|
||||||
|
// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in
|
||||||
|
// step by hand, and moving the store is an ordinary reassignment rather than an edit on every
|
||||||
|
// build machine.
|
||||||
|
//
|
||||||
|
// The environment variable remains for a builder run by a person, which is how this started and
|
||||||
|
// how it is still run while being developed.
|
||||||
|
func whereToPublish() (string, error) {
|
||||||
|
binding := strings.TrimSpace(os.Getenv("MESH_BINDING"))
|
||||||
|
if binding == "" {
|
||||||
|
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
|
||||||
|
if registry == "" {
|
||||||
|
return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " +
|
||||||
|
"nobody can fetch is not built")
|
||||||
|
}
|
||||||
|
return registry, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := os.ReadFile(binding)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err)
|
||||||
|
}
|
||||||
|
var told struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
At string `json:"at"`
|
||||||
|
Serves map[string]any `json:"serves"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &told); err != nil {
|
||||||
|
return "", fmt.Errorf("%s is not a binding: %w", binding, err)
|
||||||
|
}
|
||||||
|
if told.At == "" {
|
||||||
|
// The provider is not on the private network, so there is no name to reach it by. Said
|
||||||
|
// rather than falling back to the machine's own name, which would publish to a store on
|
||||||
|
// the wrong machine and be found out much later.
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"%s says the artifact store is on %q and gives no address for it", binding, told.From)
|
||||||
|
}
|
||||||
|
port, ok := told.Serves["port"]
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("%s says nothing about which port the artifact store answers on",
|
||||||
|
binding)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s:%v", told.At, port), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// brokerFrom is where this builder connects, and with what.
|
||||||
|
//
|
||||||
|
// **Preferably from a file the mesh sealed to this machine.** A builder that is a module is given
|
||||||
|
// its credential the way every other module is given one: generated or accepted centrally, sealed
|
||||||
|
// to the machine, written by the host. Putting it in an environment variable instead would mean
|
||||||
|
// the one copy that matters passing through a terminal and a process listing.
|
||||||
|
//
|
||||||
|
// The variable remains for a builder run by a person.
|
||||||
|
func brokerFrom() (Credential, error) {
|
||||||
|
if path := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); path != "" {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return Credential{}, fmt.Errorf("cannot read this builder's credential: %w", err)
|
||||||
|
}
|
||||||
|
said := strings.TrimSpace(string(raw))
|
||||||
|
if said == "" {
|
||||||
|
// An empty credential file is a machine that will connect as nobody and be refused,
|
||||||
|
// with the reason three layers away.
|
||||||
|
return Credential{}, fmt.Errorf("%s is empty, so this builder has no credential", path)
|
||||||
|
}
|
||||||
|
var held Credential
|
||||||
|
if err := json.Unmarshal([]byte(said), &held); err == nil && held.URL != "" {
|
||||||
|
return held, nil
|
||||||
|
}
|
||||||
|
// A file holding only a URL, which is what a person writing one by hand produces. The
|
||||||
|
// broker is then verified against whatever this machine already trusts.
|
||||||
|
return Credential{URL: said}, nil
|
||||||
|
}
|
||||||
|
return Credential{}, fmt.Errorf(
|
||||||
|
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Credential is what a build machine is given so it can reach the broker.
|
||||||
|
//
|
||||||
|
// Two things, because reaching a broker over TLS needs both: who to connect as, and what to check
|
||||||
|
// the certificate against. A mesh's broker presents a certificate of the mesh's own, which is in
|
||||||
|
// no public trust store, so a URL alone can only connect to a broker somebody else vouches for.
|
||||||
|
//
|
||||||
|
// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels
|
||||||
|
// out of band — here, sealed with the credential — and the endpoint is verified once at connect.
|
||||||
|
type Credential struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
// User and Password ride beside the address on the bus being built (design 25): a credential
|
||||||
|
// embedded in a URL leaks into every log line that prints a connection, so the mesh seals them
|
||||||
|
// as two fields and this machine joins them once, here, to dial.
|
||||||
|
User string `json:"user,omitempty"`
|
||||||
|
Password string `json:"password,omitempty"`
|
||||||
|
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||||
|
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||||
|
// serves; a credential naming none is from before claims travelled in it.
|
||||||
|
Claims []struct {
|
||||||
|
Seat string `json:"seat"`
|
||||||
|
} `json:"claims,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatsClaimed is the seats the credential names, in order.
|
||||||
|
func (c Credential) seatsClaimed() []string {
|
||||||
|
out := make([]string, 0, len(c.Claims))
|
||||||
|
for _, claim := range c.Claims {
|
||||||
|
out = append(out, claim.Seat)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||||
|
// mesh only ever seals such a credential with the user and password beside it.
|
||||||
|
func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") }
|
||||||
|
|
||||||
|
// natsURL is the address with this machine's credential in it, for the one dial that needs it.
|
||||||
|
func (c Credential) natsURL() string {
|
||||||
|
rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://")
|
||||||
|
if c.User == "" {
|
||||||
|
return "nats://" + rest
|
||||||
|
}
|
||||||
|
return "nats://" + c.User + ":" + c.Password + "@" + rest
|
||||||
|
}
|
||||||
@@ -0,0 +1,181 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildOnce is the builder doing one build and stopping, with no broker and no mesh.
|
||||||
|
//
|
||||||
|
// **This is how a mesh is raised** (novox/hq ADR 0073). The installer carries this program and runs
|
||||||
|
// it once, before anything else exists, to produce the control plane from the same repository and
|
||||||
|
// path that every later rebuild of it will use. What raises the mesh is therefore the same thing
|
||||||
|
// that maintains it — not a second mechanism that has to be kept in step with the first and is
|
||||||
|
// exercised once per new mesh, which is how often enough to rot.
|
||||||
|
//
|
||||||
|
// It takes no work from a queue and answers nobody: there is no broker yet, and the only thing
|
||||||
|
// waiting for the answer is the installer that started it. So the result goes to standard output as
|
||||||
|
// JSON, which is what the installer reads.
|
||||||
|
//
|
||||||
|
// With no --registry it publishes nowhere and the image stays in this machine's container runtime,
|
||||||
|
// named by the digest of its own configuration (see builder.Local). That is the genesis case. With
|
||||||
|
// one, it publishes as it always does — the same command is how an operator builds a module by hand
|
||||||
|
// on a mesh that already exists.
|
||||||
|
func buildOnce(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||||
|
path := set.String("path", "", "the module's directory inside the repository (default: its root)")
|
||||||
|
ref := set.String("ref", "", "the commit to build; a branch is a moving target somebody else controls")
|
||||||
|
registry := set.String("registry", "",
|
||||||
|
"host:port to publish to. Without it the artifacts stay in this machine's container runtime, which is the genesis case")
|
||||||
|
workspace := set.String("workspace", "", "where to clone and build (default: a temporary directory)")
|
||||||
|
on := multiple(set, "on",
|
||||||
|
"a base this build stands on, as <module>/<artifact>=<reference>. Repeatable")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]")
|
||||||
|
}
|
||||||
|
repository := positionals[0]
|
||||||
|
|
||||||
|
where := *workspace
|
||||||
|
if where == "" {
|
||||||
|
where = os.TempDir() + "/mesh-builder-once"
|
||||||
|
}
|
||||||
|
|
||||||
|
// What this build may stand on, said on the command line because there is no mesh to ask.
|
||||||
|
//
|
||||||
|
// **This is the genesis case and it is meant to be awkward.** On a running mesh the control
|
||||||
|
// plane answers this, because only it knows what this mesh holds. Here nothing has been built
|
||||||
|
// yet, so whoever runs this says what to use — and for the control plane, which is the one
|
||||||
|
// module raised before anything else exists, the answer is ordinarily nothing at all.
|
||||||
|
bases := map[string]string{}
|
||||||
|
for _, pair := range *on {
|
||||||
|
key, reference, found := strings.Cut(pair, "=")
|
||||||
|
if !found || key == "" || reference == "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"--on takes <module>/<artifact>=<reference>, and %q is not that", pair)
|
||||||
|
}
|
||||||
|
bases[key] = reference
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local unless told otherwise, because the moment this exists for has nowhere to publish. A
|
||||||
|
// default pointing at a registry would mean genesis failing at a push to something that is not
|
||||||
|
// there yet, one step away from the thing that could explain it.
|
||||||
|
var publisher builder.Publisher = builder.Local{Run: builder.Command}
|
||||||
|
if *registry != "" {
|
||||||
|
publisher = builder.Registry{Address: *registry, Run: builder.Command}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(os.Stderr, "building %s", repository)
|
||||||
|
if *path != "" {
|
||||||
|
fmt.Fprintf(os.Stderr, " at %s", *path)
|
||||||
|
}
|
||||||
|
if *ref != "" {
|
||||||
|
fmt.Fprintf(os.Stderr, " at %s", *ref)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(os.Stderr)
|
||||||
|
|
||||||
|
npmrc, err := packagesFrom()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
||||||
|
forgeFrom(),
|
||||||
|
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
||||||
|
if buildErr != nil {
|
||||||
|
return buildErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// To standard output, and everything else to standard error, so the caller can read this
|
||||||
|
// without having to separate it from progress.
|
||||||
|
out := onceResult{
|
||||||
|
Module: built.Manifest.Module,
|
||||||
|
Commit: built.Commit,
|
||||||
|
Repository: repository,
|
||||||
|
Path: *path,
|
||||||
|
Ref: *ref,
|
||||||
|
Manifest: built.Manifest,
|
||||||
|
Against: built.Against,
|
||||||
|
}
|
||||||
|
for _, r := range built.Read {
|
||||||
|
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
|
}
|
||||||
|
for _, made := range built.Built {
|
||||||
|
out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference})
|
||||||
|
}
|
||||||
|
body, err := json.MarshalIndent(out, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// onceResult is what one build reports to whoever started it.
|
||||||
|
//
|
||||||
|
// The same fields the mesh records for a build, so a reader comparing a genesis build against an
|
||||||
|
// ordinary one is comparing the same thing said the same way.
|
||||||
|
type onceResult struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Commit string `json:"commit"`
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Path string `json:"path,omitempty"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
Manifest any `json:"manifest"`
|
||||||
|
Made []madeArtifact `json:"made"`
|
||||||
|
Against []string `json:"against,omitempty"`
|
||||||
|
Read []readRepository `json:"read,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readRepository is a repository this build read source from besides the module's own.
|
||||||
|
type readRepository struct {
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Ref string `json:"ref,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type madeArtifact struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
Reference string `json:"reference"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseAround lets flags appear on either side of the repository, because a person writing this by
|
||||||
|
// hand will put them wherever reads best and the standard parser stops at the first thing that is
|
||||||
|
// not a flag. The same helper the control plane's commands use, for the same reason.
|
||||||
|
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||||
|
var positionals []string
|
||||||
|
rest := args
|
||||||
|
for {
|
||||||
|
if err := set.Parse(rest); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rest = set.Args()
|
||||||
|
if len(rest) == 0 {
|
||||||
|
return positionals, nil
|
||||||
|
}
|
||||||
|
positionals = append(positionals, rest[0])
|
||||||
|
rest = rest[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// multiple is a flag that may be given more than once.
|
||||||
|
type repeated []string
|
||||||
|
|
||||||
|
func (r *repeated) String() string { return strings.Join(*r, ", ") }
|
||||||
|
func (r *repeated) Set(v string) error { *r = append(*r, v); return nil }
|
||||||
|
|
||||||
|
func multiple(set *flag.FlagSet, name, usage string) *[]string {
|
||||||
|
var values repeated
|
||||||
|
set.Var(&values, name, usage)
|
||||||
|
return (*[]string)(&values)
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||||
|
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||||
|
var held Credential
|
||||||
|
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||||
|
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||||
|
t.Errorf("the old builder's credential serves %q", got)
|
||||||
|
}
|
||||||
|
var bare Credential
|
||||||
|
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||||
|
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **The genesis path writes its result to stdout and nothing else there.** The installer captures
|
||||||
|
// stdout alone and parses it as JSON; one stray log line makes that fail with "invalid character"
|
||||||
|
// — exactly what a builder fmt.Printf caused, breaking a build that had worked. This keeps
|
||||||
|
// diagnostics on stderr so a future print cannot repeat it silently.
|
||||||
|
func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||||
|
allowed := map[string]bool{
|
||||||
|
"string(body)": true, // once.go: the result JSON, which IS stdout
|
||||||
|
"version)": true, // --version
|
||||||
|
`"stopping")`: true, // the loop.s shutdown line
|
||||||
|
"usage)": true, // --help text, for a human
|
||||||
|
}
|
||||||
|
for _, file := range []string{"once.go", "main.go"} {
|
||||||
|
src, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i, raw := range strings.Split(string(src), "\n") {
|
||||||
|
line := strings.TrimSpace(raw)
|
||||||
|
if !strings.HasPrefix(line, "fmt.Printf(") &&
|
||||||
|
!strings.HasPrefix(line, "fmt.Println(") &&
|
||||||
|
!strings.HasPrefix(line, "fmt.Print(") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ok := false
|
||||||
|
for token := range allowed {
|
||||||
|
if strings.Contains(line, token) {
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Errorf("%s:%d writes a diagnostic to stdout, which the installer parses as JSON:\n %s",
|
||||||
|
file, i+1, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"encoding/hex"
|
||||||
|
"math/big"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Where a builder publishes.
|
||||||
|
//
|
||||||
|
// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh
|
||||||
|
// writes it a binding saying which machine answers and on what port. Reading it means the address
|
||||||
|
// is not a setting somebody keeps in step by hand.
|
||||||
|
|
||||||
|
func binding(t *testing.T, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "artifact-store.json")
|
||||||
|
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheMeshSaysWhereToPublish(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store",
|
||||||
|
"from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`))
|
||||||
|
where, err := whereToPublish()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if where != "anchor.internal:5000" {
|
||||||
|
t.Fatalf("got %q", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABindingWithNoAddressIsRefused(t *testing.T) {
|
||||||
|
// The provider is not on the private network, so there is no name to reach it by. Falling
|
||||||
|
// back to anything would publish to a store on the wrong machine and be found out much later.
|
||||||
|
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`))
|
||||||
|
_, err := whereToPublish()
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a binding with nowhere to reach was accepted")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "anchor") {
|
||||||
|
t.Fatalf("the failure does not name the machine: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABindingWithNoPortIsRefused(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`))
|
||||||
|
if _, err := whereToPublish(); err == nil {
|
||||||
|
t.Fatal("a binding saying nothing about a port was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) {
|
||||||
|
// Which is how this started and how it is still run while being developed.
|
||||||
|
t.Setenv("MESH_BINDING", "")
|
||||||
|
t.Setenv("MESH_REGISTRY", "127.0.0.1:5000")
|
||||||
|
where, err := whereToPublish()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if where != "127.0.0.1:5000" {
|
||||||
|
t.Fatalf("got %q", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BINDING", "")
|
||||||
|
t.Setenv("MESH_REGISTRY", "")
|
||||||
|
if _, err := whereToPublish(); err == nil {
|
||||||
|
t.Fatal("a builder with nowhere to publish reported somewhere")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheCredentialComesFromAFileTheMeshSealed(t *testing.T) {
|
||||||
|
// A builder that is a module is given its credential the way every module is: sealed to the
|
||||||
|
// machine and written by the host. An environment variable instead would put the one copy
|
||||||
|
// that matters through a terminal and a process listing.
|
||||||
|
path := filepath.Join(t.TempDir(), "broker")
|
||||||
|
if err := os.WriteFile(path, []byte("amqps://a-builder:secret@broker.internal:5671/\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_BROKER_FILE", path)
|
||||||
|
t.Setenv("MESH_BROKER_AMQP", "amqp://should-not-be-used@nowhere/")
|
||||||
|
|
||||||
|
got, err := brokerFrom()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
||||||
|
t.Fatalf("got %q", got.URL)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The credential the mesh seals carries what to check the broker's certificate against, because a
|
||||||
|
// mesh's broker presents a certificate of the mesh's own and no public trust store has it. A URL
|
||||||
|
// alone can only reach a broker somebody else vouches for.
|
||||||
|
func TestTheSealedCredentialCarriesWhatVerifiesTheBroker(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "broker")
|
||||||
|
if err := os.WriteFile(path, []byte(
|
||||||
|
`{"url":"amqps://a-builder:secret@broker.internal:5671/","fingerprint":"abc123"}`),
|
||||||
|
0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_BROKER_FILE", path)
|
||||||
|
t.Setenv("MESH_BROKER_AMQP", "")
|
||||||
|
|
||||||
|
got, err := brokerFrom()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
||||||
|
t.Fatalf("the url was lost: %q", got.URL)
|
||||||
|
}
|
||||||
|
if got.Fingerprint != "abc123" {
|
||||||
|
t.Fatal("the builder was given nothing to check the broker against, so it can only " +
|
||||||
|
"connect to a broker some public authority vouches for")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEmptyCredentialFileIsRefused(t *testing.T) {
|
||||||
|
// Otherwise the builder connects as nobody and is refused, with the reason three layers away.
|
||||||
|
path := filepath.Join(t.TempDir(), "broker")
|
||||||
|
if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_BROKER_FILE", path)
|
||||||
|
t.Setenv("MESH_BROKER_AMQP", "")
|
||||||
|
if _, err := brokerFrom(); err == nil {
|
||||||
|
t.Fatal("an empty credential was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABuilderWithNoCredentialAtAllSaysSo(t *testing.T) {
|
||||||
|
t.Setenv("MESH_BROKER_FILE", "")
|
||||||
|
t.Setenv("MESH_BROKER_AMQP", "")
|
||||||
|
if _, err := brokerFrom(); err == nil {
|
||||||
|
t.Fatal("a builder with no broker reported one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The pin is compared in the spelling the mesh writes it.
|
||||||
|
//
|
||||||
|
// A bare digest against a written fingerprint never matches, and the failure is indistinguishable
|
||||||
|
// from being pointed at the wrong broker — which is the one thing this check exists to report
|
||||||
|
// truthfully. It cost a lab run.
|
||||||
|
func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
|
||||||
|
certificate, key := aServerCertificate(t)
|
||||||
|
der := certificate.Certificate[0]
|
||||||
|
sum := sha256.Sum256(der)
|
||||||
|
written := "sha256:" + hex.EncodeToString(sum[:])
|
||||||
|
|
||||||
|
listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
|
||||||
|
Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer listener.Close()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_ = conn.(*tls.Conn).Handshake()
|
||||||
|
conn.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
_ = key
|
||||||
|
|
||||||
|
// The pin the mesh wrote must be accepted.
|
||||||
|
if err := handshakeWith(listener.Addr().String(), written); err != nil {
|
||||||
|
t.Fatalf("the broker this builder was told about was refused: %v", err)
|
||||||
|
}
|
||||||
|
// And a different one refused, or the check reports nothing.
|
||||||
|
other := "sha256:" + strings.Repeat("ab", 32)
|
||||||
|
if err := handshakeWith(listener.Addr().String(), other); err == nil {
|
||||||
|
t.Fatal("a broker this builder was not told about was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handshakeWith runs the pin check the builder dials with against an address.
|
||||||
|
func handshakeWith(address, pin string) error {
|
||||||
|
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return conn.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func aServerCertificate(t *testing.T) (tls.Certificate, ed25519.PrivateKey) {
|
||||||
|
t.Helper()
|
||||||
|
public, private, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
template := &x509.Certificate{
|
||||||
|
SerialNumber: big.NewInt(1),
|
||||||
|
Subject: pkix.Name{CommonName: "a broker"},
|
||||||
|
NotBefore: time.Now().Add(-time.Hour),
|
||||||
|
NotAfter: time.Now().Add(time.Hour),
|
||||||
|
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||||
|
}
|
||||||
|
der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: private}, private
|
||||||
|
}
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The things the mesh can be asked to do, separated from how it was asked.
|
||||||
|
//
|
||||||
|
// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and
|
||||||
|
// the command API call the same functions here, so an assignment refused at one is refused at the
|
||||||
|
// other for the same reason and in the same words. The moment a surface can accept something
|
||||||
|
// another would reject, the mesh has two answers to one question and people learn which to trust.
|
||||||
|
//
|
||||||
|
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
|
||||||
|
// composes its own explanation, because two explanations of one refusal drift.
|
||||||
|
|
||||||
|
// assign puts a module on a node, and says at once what in the mesh no longer works out.
|
||||||
|
//
|
||||||
|
// The assignment is kept even when the node does not resolve: it is what a person meant, and
|
||||||
|
// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
|
||||||
|
// long as it takes to assign the provider, and refusing the first half of a pair would make the
|
||||||
|
// order somebody types two commands in part of the mesh's rules.
|
||||||
|
//
|
||||||
|
// **What is checked is the mesh, not the one machine** — because that is what the assignment
|
||||||
|
// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
|
||||||
|
// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
|
||||||
|
// action tested one node and the verify is resolution over all of them, so an assignment could be
|
||||||
|
// reported as fine while it took a provision away from every other machine — a module offering a
|
||||||
|
// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
|
||||||
|
// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
|
||||||
|
// module already assigned. That was found on a four-node raise and read as a version bump breaking
|
||||||
|
// provider recognition; it was neither the version nor the provider.
|
||||||
|
//
|
||||||
|
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||||
|
// machines this just blocked is worth more than the milliseconds.
|
||||||
|
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||||
|
if len(modules) == 0 {
|
||||||
|
return "", fmt.Errorf("assign %s names no module", node)
|
||||||
|
}
|
||||||
|
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||||
|
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||||
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||||
|
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||||
|
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||||
|
// — the service manager, the package manager and the runtime before anything that installs,
|
||||||
|
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||||
|
// holders that depend on each other go on in one command.
|
||||||
|
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||||
|
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||||
|
// assignment resolves against a record rather than against a coincidence.
|
||||||
|
settled, err := recordDerivedHolders(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var lines []string
|
||||||
|
var added []string
|
||||||
|
for _, module := range modules {
|
||||||
|
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||||
|
if err != nil {
|
||||||
|
return strings.Join(lines, "\n"), err
|
||||||
|
}
|
||||||
|
if !fresh {
|
||||||
|
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||||
|
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||||
|
lines = append(lines, fmt.Sprintf(
|
||||||
|
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
added = append(added, module)
|
||||||
|
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||||
|
}
|
||||||
|
if len(added) == 0 {
|
||||||
|
return strings.Join(lines, "\n"), nil
|
||||||
|
}
|
||||||
|
answer := strings.Join(lines, "\n")
|
||||||
|
for _, line := range settled {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||||
|
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||||
|
// node's list, and every other node's, is `status`'s.
|
||||||
|
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||||
|
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||||
|
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||||
|
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||||
|
for _, module := range added {
|
||||||
|
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
}
|
||||||
|
plan, _, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||||
|
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||||
|
return answer + blockedElsewhere(ctx, open, node), err
|
||||||
|
}
|
||||||
|
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||||
|
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||||
|
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||||
|
isAdded := map[string]bool{}
|
||||||
|
for _, m := range added {
|
||||||
|
isAdded[m] = true
|
||||||
|
}
|
||||||
|
for _, u := range plan.Unhostable {
|
||||||
|
if !isAdded[u.Module] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range u.Missing {
|
||||||
|
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||||
|
blockedElsewhere(ctx, open, node), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||||
|
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||||
|
// and are not judged again.
|
||||||
|
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||||
|
map[string]catalogue.Manifest, []string, error) {
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
already := map[string]bool{}
|
||||||
|
for _, a := range assigned {
|
||||||
|
already[a] = true
|
||||||
|
}
|
||||||
|
var adding []string
|
||||||
|
for _, m := range modules {
|
||||||
|
if !already[m] {
|
||||||
|
adding = append(adding, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||||
|
// with everything else this act changed, so they are not said twice.
|
||||||
|
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||||
|
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||||
|
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return shelf, assigned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||||
|
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||||
|
//
|
||||||
|
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||||
|
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||||
|
// about the command's own output would ever have said so.
|
||||||
|
//
|
||||||
|
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||||
|
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||||
|
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||||
|
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||||
|
if len(modules) == 0 {
|
||||||
|
return "", fmt.Errorf("unassign %s names no module", node)
|
||||||
|
}
|
||||||
|
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||||
|
runs := map[string]bool{}
|
||||||
|
for _, a := range assigned {
|
||||||
|
runs[a] = true
|
||||||
|
}
|
||||||
|
for _, module := range modules {
|
||||||
|
if !runs[module] {
|
||||||
|
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, module := range modules {
|
||||||
|
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||||
|
node, strings.Join(modules, ", "), node)
|
||||||
|
var left []string
|
||||||
|
for _, a := range assigned {
|
||||||
|
if !slices.Contains(modules, a) {
|
||||||
|
left = append(left, a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||||
|
answer += "\n " + line
|
||||||
|
}
|
||||||
|
return answer + blockedElsewhere(ctx, open, node), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||||
|
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||||
|
// command API and the controller seat's verbs as they do from the command line.
|
||||||
|
func splitModules(field string) []string {
|
||||||
|
var out []string
|
||||||
|
for _, m := range strings.Split(field, ",") {
|
||||||
|
if m = strings.TrimSpace(m); m != "" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||||
|
//
|
||||||
|
// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
|
||||||
|
// whole mesh twice and would still be a guess about which of several changes did it; saying
|
||||||
|
// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
|
||||||
|
// and cannot mislead. The machine that was just changed is left out because its own refusal is
|
||||||
|
// already the answer beside this one.
|
||||||
|
//
|
||||||
|
// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
|
||||||
|
// not a reason to claim the assignment did not happen — it did.
|
||||||
|
func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||||
|
nodes, err := open.inventory.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "\n\nThe rest of the mesh could not be checked: " + err.Error()
|
||||||
|
}
|
||||||
|
blocked := map[string]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.Name == except {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, _, err := planFor(ctx, open, n.Name); err != nil {
|
||||||
|
blocked[n.Name] = err.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(blocked) == 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(blocked))
|
||||||
|
for name := range blocked {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
var out strings.Builder
|
||||||
|
fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
|
||||||
|
"nothing will be sent to them:\n", len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
fmt.Fprintf(&out, " %s\n", name)
|
||||||
|
for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
|
||||||
|
fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||||
|
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||||
|
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||||
|
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||||
|
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||||
|
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||||
|
inv := open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
m, known := shelf[module]
|
||||||
|
if !known || mayIssue(m) != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||||
|
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
busAddress, err := broker.BusAddress()
|
||||||
|
if err == nil {
|
||||||
|
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||||
|
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What an assignment says about the machines it was not about.
|
||||||
|
|
||||||
|
// **An assignment that blocks another machine says so.**
|
||||||
|
//
|
||||||
|
// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it
|
||||||
|
// the moment its own node stops resolving, so an assignment to the provider's machine silently took
|
||||||
|
// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this
|
||||||
|
// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the
|
||||||
|
// way back, as a version bump breaking provider recognition. It was neither the version nor the
|
||||||
|
// provider: it was one machine's set of assignments, and nothing said so.
|
||||||
|
//
|
||||||
|
// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify
|
||||||
|
// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the
|
||||||
|
// verify is resolution over all of them.
|
||||||
|
func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
|
||||||
|
// A provider on the hub and a consumer on the other machine, both resolving.
|
||||||
|
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||||
|
Requires: []string{"acme-ca"}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := assign(ctx, open, "laptop", "route-proxy")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
if strings.Contains(said, "cannot be worked out") {
|
||||||
|
t.Fatalf("a well mesh was reported as blocked:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now break the hub's own set, with nothing but the command a person has.
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err = assign(ctx, open, "anchor", "rival-two")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("an assignment that makes its own node incoherent was not reported at all")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The node's own refusal is the error, as it always was. What is new is that the machines this
|
||||||
|
// just took a provision away from are named in the same breath.
|
||||||
|
if !strings.Contains(said, "laptop") {
|
||||||
|
t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "acme-ca") {
|
||||||
|
t.Fatalf("what laptop is now missing is not said:\n%s", said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "cannot be worked out as things stand") {
|
||||||
|
t.Fatalf("the report does not say what state the mesh is in:\n%s", said)
|
||||||
|
}
|
||||||
|
// And the assignment is kept: it is what a person meant, and assignment is not an ordering.
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !contains(assigned, "rival-two") {
|
||||||
|
t.Fatalf("the assignment was not kept: %v", assigned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A consumer assigned before its provider is refused for itself and kept, because assignment is not
|
||||||
|
// an ordering — refusing the first half of a pair would make the order somebody types two commands
|
||||||
|
// in part of the mesh's rules.
|
||||||
|
func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||||
|
Requires: []string{"acme-ca"}})
|
||||||
|
|
||||||
|
said, err := assign(ctx, open, "laptop", "route-proxy")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("a consumer with nothing to consume resolved:\n%s", said)
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !contains(assigned, "route-proxy") {
|
||||||
|
t.Fatalf("assignment became an ordering: %v", assigned)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unassigning is the ordinary way to stop providing something to another machine, and it reports
|
||||||
|
// the same way for the same reason.
|
||||||
|
func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||||
|
Requires: []string{"acme-ca"}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := unassign(ctx, open, "anchor", "step-ca")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") {
|
||||||
|
t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(all []string, one string) bool {
|
||||||
|
for _, s := range all {
|
||||||
|
if s == one {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,344 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An address is read from the node's settings where it is used, never recorded with a port
|
||||||
|
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
||||||
|
|
||||||
|
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||||
|
|
||||||
|
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
|
||||||
|
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
|
||||||
|
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||||
|
manifest, _ := json.Marshal(map[string]any{
|
||||||
|
"module": "gitea", "version": "1",
|
||||||
|
"resources": []map[string]any{
|
||||||
|
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||||
|
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
||||||
|
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
||||||
|
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||||
|
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
||||||
|
"image": "valkey/valkey@" + aDigest},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
kept := buildFrom(link.BuildResult{
|
||||||
|
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
||||||
|
Manifest: manifest,
|
||||||
|
Made: []link.MadeArtifact{
|
||||||
|
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
||||||
|
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||||
|
},
|
||||||
|
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
||||||
|
})
|
||||||
|
if kept.Module != "gitea" {
|
||||||
|
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
||||||
|
}
|
||||||
|
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||||
|
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
||||||
|
}
|
||||||
|
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||||
|
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
||||||
|
}
|
||||||
|
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||||
|
t.Errorf("the recorded manifest's image is %v", got)
|
||||||
|
}
|
||||||
|
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||||
|
t.Errorf("the recorded manifest's archive is %v", got)
|
||||||
|
}
|
||||||
|
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
||||||
|
t.Errorf("an image the build did not make was rewritten: %v", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||||
|
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||||
|
}
|
||||||
|
// What the build stood on is an edge to another module's artifact, and it is recorded the way
|
||||||
|
// that artifact is: by path in the store, so the edge still names the same thing when the
|
||||||
|
// store answers at another address.
|
||||||
|
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
|
||||||
|
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||||
|
// distribution module does, so a node may be given another number for it.
|
||||||
|
func aStore() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "distribution", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
||||||
|
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||||
|
// through, say the port the node gave the store** — not the catalogue's number.
|
||||||
|
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, aStore())
|
||||||
|
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||||
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||||
|
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||||
|
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||||
|
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||||
|
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||||
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
on := map[string]bool{"anchor": true, "laptop": true}
|
||||||
|
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||||
|
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||||
|
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var trust string
|
||||||
|
for _, r := range composed(t, open, "laptop").Resources {
|
||||||
|
if r["path"] == "/etc/docker/daemon.json" {
|
||||||
|
trust, _ = r["content"].(string)
|
||||||
|
}
|
||||||
|
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||||
|
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||||
|
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a replay to the catalogue says where the store is now.
|
||||||
|
announced, err := following{open}.Announceable(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||||
|
t.Fatalf("the replay announces %+v", announced)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||||
|
//
|
||||||
|
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||||
|
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||||
|
// process is told so beside the sealed connection genesis wrote.
|
||||||
|
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
raw, err := os.ReadFile("../../module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||||
|
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, control)
|
||||||
|
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||||
|
{Port: 5672, From: catalogue.FromMesh}},
|
||||||
|
Guards: []int{15672},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||||
|
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||||
|
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||||
|
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||||
|
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||||
|
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||||
|
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var env map[string]any
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "mesh-controller.controller" {
|
||||||
|
env, _ = r["env"].(map[string]any)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if env == nil {
|
||||||
|
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||||
|
}
|
||||||
|
for key, want := range map[string]string{
|
||||||
|
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||||
|
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||||
|
"MESH_STORE_LICENCES_PORT": "6852",
|
||||||
|
"MESH_BROKER_AMQP_PORT": "5679",
|
||||||
|
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||||
|
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||||
|
"MESH_BROKER_ADDRESS_PORT": "",
|
||||||
|
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||||
|
} {
|
||||||
|
if env[key] != want {
|
||||||
|
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||||
|
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||||
|
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
||||||
|
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||||
|
seatPorts := map[string]string{
|
||||||
|
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||||
|
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||||
|
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||||
|
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||||
|
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||||
|
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||||
|
}
|
||||||
|
out := m
|
||||||
|
out.Resources = nil
|
||||||
|
for _, r := range m.Resources {
|
||||||
|
if r["type"] != "container" && r["type"] != "process" {
|
||||||
|
out.Resources = append(out.Resources, r)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
copied := map[string]any{}
|
||||||
|
for k, v := range r {
|
||||||
|
copied[k] = v
|
||||||
|
}
|
||||||
|
env := map[string]any{}
|
||||||
|
if had, ok := r["env"].(map[string]any); ok {
|
||||||
|
for k, v := range had {
|
||||||
|
env[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k, v := range seatPorts {
|
||||||
|
if _, said := env[k]; !said {
|
||||||
|
env[k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
copied["env"] = env
|
||||||
|
out.Resources = append(out.Resources, copied)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||||
|
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||||
|
// catalogue have each been built and pushed (finding F2).
|
||||||
|
func aLoneNode(t *testing.T) *stores {
|
||||||
|
t.Helper()
|
||||||
|
inventory.ForTest(t)
|
||||||
|
licences.ForTest(t)
|
||||||
|
open, err := openStores(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(open.Close)
|
||||||
|
for _, m := range provided {
|
||||||
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||||
|
{"name": "container-runtime", "present": true}}})
|
||||||
|
var profile map[string]any
|
||||||
|
_ = json.Unmarshal(reported, &profile)
|
||||||
|
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||||
|
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||||
|
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||||
|
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||||
|
open := aLoneNode(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, aStore())
|
||||||
|
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||||
|
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||||
|
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||||
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||||
|
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||||
|
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||||
|
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||||
|
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||||
|
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||||
|
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var image any
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "postgres.runtime" {
|
||||||
|
image = r["image"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||||
|
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||||
|
}
|
||||||
|
held := heldBy(ctx)
|
||||||
|
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||||
|
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,641 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and
|
||||||
|
// refused while a found container is held; returning to adopted keeps what was taken.
|
||||||
|
|
||||||
|
// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also
|
||||||
|
// runs and a module with only a file, and a filter module in the catalogue.
|
||||||
|
func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
||||||
|
t.Helper()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"},
|
||||||
|
{"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"},
|
||||||
|
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||||
|
}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"},
|
||||||
|
}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "nftables", Version: "1",
|
||||||
|
Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"},
|
||||||
|
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||||
|
"state": "running", "restart-on": []any{"filtering"}}}})
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"hello-web", "notes"} {
|
||||||
|
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sent := &[]string{}
|
||||||
|
saved := sendNodes
|
||||||
|
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||||
|
*sent = append(*sent, names...)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendNodes = saved })
|
||||||
|
return open, sent
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportsHolding has the anchor report, on what it was last sent, holding what is given.
|
||||||
|
func reportsHolding(t *testing.T, open *stores, held ...link.Held) {
|
||||||
|
t.Helper()
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry",
|
||||||
|
Published: true, ContainerPort: 5000},
|
||||||
|
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker",
|
||||||
|
Published: true, ContainerPort: 15672},
|
||||||
|
}, held...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and
|
||||||
|
// holding what is given.
|
||||||
|
func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) {
|
||||||
|
t.Helper()
|
||||||
|
ctx := t.Context()
|
||||||
|
body, err := composed(t, open, "anchor").Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
record, err := open.inventory.NodeByName(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||||
|
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||||
|
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||||
|
// A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a
|
||||||
|
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
|
||||||
|
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
|
||||||
|
Outward: []string{"eth0"},
|
||||||
|
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
|
||||||
|
// predecessor left in the runtime's user chain.
|
||||||
|
Filters: anchorFilters,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
|
||||||
|
// predecessor's chain the mesh did not write.
|
||||||
|
var anchorFilters = []link.Filter{
|
||||||
|
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
|
||||||
|
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
|
||||||
|
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||||
|
Target: "hello-web", Since: time.Now()}
|
||||||
|
heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file",
|
||||||
|
Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"}
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||||
|
t.Fatalf("taking an unassigned module gave %v", err)
|
||||||
|
}
|
||||||
|
if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||||
|
t.Fatalf("taking on a converged node gave %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
_, err := converge(t.Context(), open, "anchor", false, "", "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has not reported") {
|
||||||
|
t.Fatalf("a node that never reported was previewed: %v", err)
|
||||||
|
}
|
||||||
|
if len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused converge sent something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") {
|
||||||
|
t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
ctx := t.Context()
|
||||||
|
// The machine holds something for the module, so the take acts on the preview the operator
|
||||||
|
// saw and names its digest (novox/hq ADR 0163).
|
||||||
|
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saw := takeDigestIn(t, preview)
|
||||||
|
if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") {
|
||||||
|
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||||
|
}
|
||||||
|
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||||
|
t.Fatal("the preview took something")
|
||||||
|
}
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") {
|
||||||
|
t.Fatalf("--yes without the digest was not refused: %v", err)
|
||||||
|
}
|
||||||
|
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||||
|
!strings.Contains(said, "push anchor") {
|
||||||
|
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeDigestIn is the digest a take's preview printed.
|
||||||
|
func takeDigestIn(t *testing.T, preview string) string {
|
||||||
|
t.Helper()
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||||
|
return fields[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// A take acts on the preview the operator saw, and on an account of the machine that is still the
|
||||||
|
// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1).
|
||||||
|
func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
reportsHolding(t, open, heldContainer, heldFile)
|
||||||
|
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saw := takeDigestIn(t, preview)
|
||||||
|
|
||||||
|
// The machine reports again, and what it holds has changed: the found container now carries
|
||||||
|
// facts the preview never showed.
|
||||||
|
changed := heldContainer
|
||||||
|
changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"}
|
||||||
|
reportsHolding(t, open, changed, heldFile)
|
||||||
|
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||||
|
t.Fatalf("a changed preview was acted on: %v", err)
|
||||||
|
}
|
||||||
|
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||||
|
t.Fatal("a refused take took something")
|
||||||
|
}
|
||||||
|
|
||||||
|
// And an account older than the flip allows.
|
||||||
|
preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saw = takeDigestIn(t, preview)
|
||||||
|
saved := reportFreshFor
|
||||||
|
reportFreshFor = -time.Second
|
||||||
|
defer func() { reportFreshFor = saved }()
|
||||||
|
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") {
|
||||||
|
t.Fatalf("a stale account was acted on: %v", err)
|
||||||
|
}
|
||||||
|
reportFreshFor = saved
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A module the machine holds nothing for has nothing to compare: --yes alone suffices.
|
||||||
|
if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil {
|
||||||
|
// notes holds a file, so this one needs the digest too.
|
||||||
|
t.Fatal("notes holds a found file and was taken without a digest")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"tcp/8080 hello-web (published, container port 80)",
|
||||||
|
"declared by hello-web (from anywhere)",
|
||||||
|
"WILL CLOSE — no module assigned here declares it",
|
||||||
|
// The anchor faces inward and is on the private network: the derived filter admits ssh
|
||||||
|
// from the mesh only.
|
||||||
|
"WILL CLOSE to everything outside the private network — ssh stays open from the mesh",
|
||||||
|
"notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at",
|
||||||
|
"assigns nftables",
|
||||||
|
"the found firewall (ufw) is disabled, never flushed",
|
||||||
|
// What it routes is not a listener: said not to be previewed, and to be dropped.
|
||||||
|
"not previewed: traffic the machine routes that is not a published port",
|
||||||
|
"the derived filter drops it unless a module declares it",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(preview, "15672") {
|
||||||
|
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||||
|
}
|
||||||
|
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
|
||||||
|
// chain is named as not the mesh's and left, so the reader knows before the flip.
|
||||||
|
for _, want := range []string{
|
||||||
|
"table ip filter, chain DOCKER-USER",
|
||||||
|
"NOT THE MESH'S; left in force",
|
||||||
|
`iifname "eth0" tcp dport 6000 drop`,
|
||||||
|
"table ip filter, chain ufw-reject-input",
|
||||||
|
"the found firewall's; retired with it",
|
||||||
|
"the container runtime's own; left",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||||
|
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("the preview changed something")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||||
|
if n.Adopted {
|
||||||
|
t.Fatal("converge --yes left the node adopted")
|
||||||
|
}
|
||||||
|
taken, _ := open.inventory.Taken(ctx, "anchor")
|
||||||
|
if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) {
|
||||||
|
t.Fatalf("the flip took %v", taken)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(*sent, []string{"anchor"}) {
|
||||||
|
t.Fatalf("the flip sent %v", *sent)
|
||||||
|
}
|
||||||
|
declared := composed(t, open, "anchor")
|
||||||
|
if declared.Adoption != nil {
|
||||||
|
t.Fatal("a converged node is still sent an adoption envelope")
|
||||||
|
}
|
||||||
|
if !hasID(declared.Resources, "nftables.filtering") {
|
||||||
|
t.Fatal("the converged node is not declared the mesh's filter")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := adopt(ctx, open, "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := adopt(ctx, open, "anchor"); err == nil {
|
||||||
|
t.Fatal("adopting an adopted node was not refused")
|
||||||
|
}
|
||||||
|
again, _ := open.inventory.Taken(ctx, "anchor")
|
||||||
|
if !reflect.DeepEqual(again, taken) {
|
||||||
|
t.Fatalf("returning to adopted lost what was taken: %v", again)
|
||||||
|
}
|
||||||
|
declared = composed(t, open, "anchor")
|
||||||
|
if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 {
|
||||||
|
t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") {
|
||||||
|
t.Fatal("returned to adopted, the mesh's filter is still declared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasID(resources []map[string]any, id string) bool {
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == id {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// The command API refuses a flip exactly as the command line does, in the same words.
|
||||||
|
func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
reportsHolding(t, open, heldContainer)
|
||||||
|
_, direct := converge(t.Context(), open, "anchor", true, "", "")
|
||||||
|
if direct == nil {
|
||||||
|
t.Fatal("the flip was not refused")
|
||||||
|
}
|
||||||
|
got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`)
|
||||||
|
if got.Code != http.StatusConflict {
|
||||||
|
t.Fatalf("got %d: %s", got.Code, got.Body.String())
|
||||||
|
}
|
||||||
|
var said map[string]any
|
||||||
|
if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if said["refused"] != direct.Error() {
|
||||||
|
t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error())
|
||||||
|
}
|
||||||
|
if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("a take naming no module got %d", got.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On
|
||||||
|
// a machine that faces inward, ssh is admitted from the private network only, and a port a module
|
||||||
|
// admits from the mesh only closes to everything outside it: both are said to close.
|
||||||
|
func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"},
|
||||||
|
{Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
})
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lines := map[string]string{}
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
fields := strings.Fields(line)
|
||||||
|
if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") {
|
||||||
|
lines[fields[0]+" "+fields[1]] += line + "\n"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+
|
||||||
|
"the private network") {
|
||||||
|
t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview)
|
||||||
|
}
|
||||||
|
store := lines["tcp/5432 postgres"]
|
||||||
|
if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 ||
|
||||||
|
!strings.Contains(store, "declared by store (from mesh)") {
|
||||||
|
t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview)
|
||||||
|
}
|
||||||
|
if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") {
|
||||||
|
t.Errorf("a port open to everywhere is not said to stay:\n%s", preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// digestIn is the digest a converge preview printed.
|
||||||
|
func digestIn(t *testing.T, preview string) string {
|
||||||
|
t.Helper()
|
||||||
|
for _, line := range strings.Split(preview, "\n") {
|
||||||
|
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||||
|
return fields[1]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused
|
||||||
|
// when the digest is missing, when anything the preview says has changed since, or when the node's
|
||||||
|
// account of itself is too old to be the machine as it is.
|
||||||
|
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saw := digestIn(t, preview)
|
||||||
|
if !strings.Contains(preview, "converge anchor --yes "+saw) {
|
||||||
|
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||||
|
}
|
||||||
|
unchanged := func() {
|
||||||
|
t.Helper()
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "--yes "+saw) {
|
||||||
|
t.Fatalf("a flip naming no preview was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
// Something new is reachable: the preview the operator saw is not what would happen.
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||||
|
ContainerPort: 80},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"},
|
||||||
|
}, heldFile)
|
||||||
|
_, err = converge(ctx, open, "anchor", true, saw, "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||||
|
t.Fatalf("a flip on a changed preview was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
// An account older than the flip trusts is refused, whatever digest is named.
|
||||||
|
again, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved := reportFreshFor
|
||||||
|
reportFreshFor = time.Nanosecond
|
||||||
|
_, err = converge(ctx, open, "anchor", true, digestIn(t, again), "")
|
||||||
|
reportFreshFor = saved
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "wait for its next report") {
|
||||||
|
t.Fatalf("a flip on an old account was not refused: %v", err)
|
||||||
|
}
|
||||||
|
unchanged()
|
||||||
|
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil {
|
||||||
|
t.Fatalf("the flip on the preview just seen was refused: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted {
|
||||||
|
t.Fatal("the flip did not converge the node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||||
|
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||||
|
// flip is not made to wait on the flip's own hold.
|
||||||
|
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var heldElsewhere, heldHere error
|
||||||
|
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||||
|
// Another caller cannot hold the node while the flip sends it.
|
||||||
|
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||||
|
release()
|
||||||
|
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||||
|
}
|
||||||
|
// The flip's own send holds it without waiting on itself.
|
||||||
|
_, release, err := holdNodes(inner, open, names)
|
||||||
|
if err != nil {
|
||||||
|
heldHere = err
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if heldElsewhere != nil || heldHere != nil {
|
||||||
|
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||||
|
}
|
||||||
|
// And given back once it is done.
|
||||||
|
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||||
|
// not only its files, and the digest changes when any of them does.
|
||||||
|
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
since := time.Now()
|
||||||
|
held := []link.Held{heldFile,
|
||||||
|
{ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since},
|
||||||
|
{ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since},
|
||||||
|
{ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since},
|
||||||
|
{ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since},
|
||||||
|
{ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since},
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, held...)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, want := range []string{
|
||||||
|
"replacing the found directory /var/lib/notes (notes.data)",
|
||||||
|
"replacing the found service notes.service (notes.daemon)",
|
||||||
|
"replacing the found archive /srv/notes (notes.seed)",
|
||||||
|
"replacing the found process notes-worker (notes.worker)",
|
||||||
|
"replacing the found user notes (notes.account)",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, held[:len(held)-1]...)
|
||||||
|
fewer, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if digestIn(t, fewer) == digestIn(t, preview) {
|
||||||
|
t.Fatal("the digest does not change with what is held")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
|
||||||
|
// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
|
||||||
|
// did not, and flipping would close ports the preview never named.
|
||||||
|
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||||
|
open, sent := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||||
|
reportsReaching(t, open, []link.Reach{
|
||||||
|
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||||
|
}, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(preview, "this account looks partial") {
|
||||||
|
t.Errorf("the preview does not mark a partial account:\n%s", preview)
|
||||||
|
}
|
||||||
|
_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
|
||||||
|
t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||||
|
t.Fatal("a refused flip changed something")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||||
|
// holds the node, so it waits for whatever is converging it.
|
||||||
|
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||||
|
open, _ := anAdoptedAnchor(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reportsHolding(t, open, heldFile)
|
||||||
|
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||||
|
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||||
|
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||||
|
|
||||||
|
var whileFlipping error
|
||||||
|
sendNodes = func(context.Context, *stores, []string) error {
|
||||||
|
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||||
|
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,186 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The command API: what the mesh can be asked to do, over a network.
|
||||||
|
//
|
||||||
|
// **An adapter and nothing else** (novox/hq ADR 0035). Every route here calls the same function
|
||||||
|
// the command line calls, so a refusal is the same refusal in the same words. Nothing is decided
|
||||||
|
// in this file — the moment it validates something the command line does not, the mesh has two
|
||||||
|
// answers to one question.
|
||||||
|
//
|
||||||
|
// **It refuses everything unless it was told how to know who is asking.** The board is published
|
||||||
|
// on a public name, and this is what stands behind it: an unauthenticated command surface reachable
|
||||||
|
// from the internet is authority over the mesh handed to whoever finds it. So there is no
|
||||||
|
// permissive default and no flag that removes the check — a mesh that has not been told how to
|
||||||
|
// authenticate serves nothing, loudly.
|
||||||
|
//
|
||||||
|
// The intended authenticator is an OAuth2 provider (ADR 0035), which is an ordinary module. Until
|
||||||
|
// one is configured this refuses, which is the correct behaviour rather than a placeholder: a
|
||||||
|
// surface that worked without authentication would be one somebody left running.
|
||||||
|
func apiCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("api", flag.ContinueOnError)
|
||||||
|
listen := set.String("listen", "127.0.0.1:8081", "where to serve it")
|
||||||
|
issuer := set.String("issuer", "",
|
||||||
|
"the OAuth2 issuer whose tokens this accepts; without it, nothing is served")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(*issuer) == "" {
|
||||||
|
// Refused at start rather than per request, so it is discovered by whoever ran it rather
|
||||||
|
// than by whoever finds it.
|
||||||
|
return errors.New(
|
||||||
|
"--issuer is not set, and this serves commands rather than pages: it will not run " +
|
||||||
|
"without being told whose tokens to believe. An OAuth2 provider is an ordinary " +
|
||||||
|
"module (novox/hq ADR 0035)")
|
||||||
|
}
|
||||||
|
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: *listen,
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
Handler: commands(mustAuthenticate(*issuer)),
|
||||||
|
}
|
||||||
|
fmt.Printf("the command API is on http://%s\n", *listen)
|
||||||
|
fmt.Printf(" it accepts tokens from %s and refuses everything else\n", *issuer)
|
||||||
|
fmt.Printf(" every route calls what the command line calls\n")
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = server.Shutdown(closing)
|
||||||
|
}()
|
||||||
|
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Authenticator says whether a request may act, and as whom.
|
||||||
|
//
|
||||||
|
// An interface so the check can be driven by a test without an identity provider, and so the one
|
||||||
|
// real implementation is the only thing that has to be right.
|
||||||
|
type Authenticator interface {
|
||||||
|
// Who returns the subject a request is acting as, or an error naming why it may not.
|
||||||
|
Who(r *http.Request) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mustAuthenticate is the real one: a bearer token from the configured issuer.
|
||||||
|
//
|
||||||
|
// **Not yet verifying the signature**, and it says so rather than pretending. Verification needs
|
||||||
|
// the issuer's keys, which needs an identity provider to exist — so this refuses every request
|
||||||
|
// until that is built, which is the same answer as having no API at all and is honest about why.
|
||||||
|
func mustAuthenticate(issuer string) Authenticator { return notYet{issuer: issuer} }
|
||||||
|
|
||||||
|
type notYet struct{ issuer string }
|
||||||
|
|
||||||
|
func (n notYet) Who(*http.Request) (string, error) {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"this mesh has no way to verify a token from %s yet: the identity provider is a module "+
|
||||||
|
"and none is running. Use the command line, which authenticates through nothing "+
|
||||||
|
"because it is already behind the machine's own login", n.issuer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// commands is the routing, separate so a test can drive it without a listener.
|
||||||
|
func commands(who Authenticator) http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
|
||||||
|
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||||
|
}))
|
||||||
|
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||||
|
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest})
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
||||||
|
}))
|
||||||
|
mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||||
|
return adopt(ctx, open, in.Node)
|
||||||
|
}))
|
||||||
|
|
||||||
|
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
||||||
|
// expected is indistinguishable from one that is down.
|
||||||
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
refuse(w, http.StatusNotFound, fmt.Errorf(
|
||||||
|
"%s %s is not something this mesh can be asked; it accepts POST /assign, "+
|
||||||
|
"POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path))
|
||||||
|
})
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
type request struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest
|
||||||
|
// of the preview it acts on, and (converge) which module loads the mesh's filter.
|
||||||
|
Yes bool `json:"yes,omitempty"`
|
||||||
|
Digest string `json:"digest,omitempty"`
|
||||||
|
Filter string `json:"filter,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// acting is the shape every route shares: authenticate, read, act, answer.
|
||||||
|
func acting(
|
||||||
|
who Authenticator,
|
||||||
|
needsModule bool,
|
||||||
|
do func(context.Context, *stores, request) (string, error),
|
||||||
|
) http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if _, err := who.Who(r); err != nil {
|
||||||
|
refuse(w, http.StatusUnauthorized, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var in request
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||||
|
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if in.Node == "" || (needsModule && in.Module == "") {
|
||||||
|
if needsModule {
|
||||||
|
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
||||||
|
} else {
|
||||||
|
refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
refuse(w, http.StatusServiceUnavailable, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
said, err := do(r.Context(), open, in)
|
||||||
|
if err != nil {
|
||||||
|
// **The refusal the command line would have given, unchanged.** Carrying `said` with
|
||||||
|
// it matters: an assignment that was kept and still does not resolve is two facts,
|
||||||
|
// and dropping either makes the answer wrong.
|
||||||
|
answer(w, http.StatusConflict, map[string]any{"said": said, "refused": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
answer(w, http.StatusOK, map[string]any{"said": said})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func answer(w http.ResponseWriter, status int, body map[string]any) {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(status)
|
||||||
|
_ = json.NewEncoder(w).Encode(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
func refuse(w http.ResponseWriter, status int, err error) {
|
||||||
|
answer(w, status, map[string]any{"refused": err.Error()})
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type letIn struct{}
|
||||||
|
|
||||||
|
func (letIn) Who(*http.Request) (string, error) { return "somebody", nil }
|
||||||
|
|
||||||
|
func asking(t *testing.T, who Authenticator, method, path, body string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
recorded := httptest.NewRecorder()
|
||||||
|
commands(who).ServeHTTP(recorded, httptest.NewRequest(method, path, strings.NewReader(body)))
|
||||||
|
return recorded
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Nothing is served to somebody the mesh cannot identify**, and that is the default rather than
|
||||||
|
// a setting. The board this stands behind is published on a public name (novox/hq 11-a-board), so
|
||||||
|
// an unauthenticated command surface is authority over the mesh handed to whoever finds it.
|
||||||
|
func TestAnUnauthenticatedRequestIsRefused(t *testing.T) {
|
||||||
|
got := asking(t, mustAuthenticate("https://identity.example/realms/mesh"),
|
||||||
|
"POST", "/assign", `{"node":"anchor","module":"umami"}`)
|
||||||
|
if got.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("an unidentified caller got %d", got.Code)
|
||||||
|
}
|
||||||
|
// And it says what to do instead, because the answer is not "give up".
|
||||||
|
if !strings.Contains(got.Body.String(), "command line") {
|
||||||
|
t.Errorf("the refusal does not say what still works: %s", got.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The API refuses to start at all without being told whose tokens to believe.
|
||||||
|
//
|
||||||
|
// At start rather than per request, so it is found by whoever ran it rather than by whoever
|
||||||
|
// finds it.
|
||||||
|
func TestTheApiWillNotRunWithoutAnIssuer(t *testing.T) {
|
||||||
|
err := apiCommand(context.Background(), []string{})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("it served commands without being told who may give them")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "issuer") {
|
||||||
|
t.Errorf("the refusal does not name what is missing: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A request missing what it acts on is refused before anything is opened.
|
||||||
|
func TestARequestThatNamesNothingIsRefused(t *testing.T) {
|
||||||
|
// **The exact refusal, not merely "not accepted".** Asserting non-200 passes even when the
|
||||||
|
// request got as far as opening a store and failing there, which proves nothing about whether
|
||||||
|
// anything was checked — that is what the first version of this test did.
|
||||||
|
for _, body := range []string{`{}`, `{"node":"anchor"}`, `{"module":"umami"}`, `not json`} {
|
||||||
|
got := asking(t, letIn{}, "POST", "/assign", body)
|
||||||
|
if got.Code != http.StatusBadRequest {
|
||||||
|
t.Errorf("%s got %d, and a request naming nothing is a bad request", body, got.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verb nobody implemented is said plainly. A command surface answering 404 to something
|
||||||
|
// somebody expected is indistinguishable from one that is down.
|
||||||
|
func TestAnUnknownRouteSaysWhatIsAccepted(t *testing.T) {
|
||||||
|
got := asking(t, letIn{}, "POST", "/rotate", `{}`)
|
||||||
|
if got.Code != http.StatusNotFound {
|
||||||
|
t.Fatalf("got %d", got.Code)
|
||||||
|
}
|
||||||
|
var said map[string]any
|
||||||
|
_ = json.Unmarshal(got.Body.Bytes(), &said)
|
||||||
|
if !strings.Contains(said["refused"].(string), "/assign") {
|
||||||
|
t.Errorf("it does not say what it does accept: %v", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ask calls one of a module's tools, through the control plane's own broker connection.
|
||||||
|
//
|
||||||
|
// A module serves tools under an account scoped to exactly that (novox/hq ADR 0047), and nothing
|
||||||
|
// else in the mesh held an account that could ask one — not an operator at a terminal, not an agent
|
||||||
|
// acting for one (novox/hq 04-ISSUES/049). The control plane does, so it is the way in: one
|
||||||
|
// process, one connection, one place a question can be seen to have been asked (ADR 0095).
|
||||||
|
func askCommand(ctx context.Context, args []string) error {
|
||||||
|
positionals, flags := split(args)
|
||||||
|
set := flag.NewFlagSet("ask", flag.ContinueOnError)
|
||||||
|
wait := set.Duration("wait", 60*time.Second, "how long to wait for the module's answer")
|
||||||
|
if err := set.Parse(flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) < 2 || len(positionals) > 3 {
|
||||||
|
return errors.New("ask <module> <tool> [json arguments] [--wait 60s]")
|
||||||
|
}
|
||||||
|
module, tool := positionals[0], positionals[1]
|
||||||
|
var arguments json.RawMessage
|
||||||
|
if len(positionals) == 3 {
|
||||||
|
if !json.Valid([]byte(positionals[2])) {
|
||||||
|
return fmt.Errorf("the arguments are not JSON: %s", positionals[2])
|
||||||
|
}
|
||||||
|
arguments = json.RawMessage(positionals[2])
|
||||||
|
}
|
||||||
|
|
||||||
|
server, err := connectLink(ctx, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The answer as the module gave it, to standard output, for a person or a program. A tool
|
||||||
|
// that answered with an error has still answered: printed the same way, and the exit status
|
||||||
|
// says which.
|
||||||
|
body, err := json.Marshal(answer)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Fprintln(os.Stdout, string(body))
|
||||||
|
if answer.Error != "" {
|
||||||
|
return fmt.Errorf("%s.%s answered with an error: %s", module, tool, answer.Error)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,324 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"html/template"
|
||||||
|
"net/http"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// boardCommand serves the three questions as a page.
|
||||||
|
//
|
||||||
|
// **It reads through the same functions everything else does and holds nothing**
|
||||||
|
// (novox/hq 03-DESIGN/01-to-be/11-a-board.md). The board being replaced is one service that reads
|
||||||
|
// every context's database directly — [ADR 0008](novox/hq) violated by the one component with a
|
||||||
|
// reason to violate it, and the cost is that a boundary nothing may cross can move, while one
|
||||||
|
// thing crossing it is enough to freeze it. A board that reads the provisioning tables is a board
|
||||||
|
// that breaks when provisioning changes its tables, and the change then gets weighed against the
|
||||||
|
// board.
|
||||||
|
//
|
||||||
|
// **It stores nothing of its own.** No cache that can disagree, no table of what the mesh looked
|
||||||
|
// like last time. Every request reads the mesh now; if that is slow, the answer belongs in the
|
||||||
|
// context that owns it, where everything else asking gets it too.
|
||||||
|
//
|
||||||
|
// **Reading is the whole of it.** Every action a board could offer already exists as a command,
|
||||||
|
// and a button that does something no command does is a second implementation of a decision.
|
||||||
|
func boardCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("board", flag.ContinueOnError)
|
||||||
|
// The private network, not everything. A board says which machines are broken and what they
|
||||||
|
// are running, which is exactly the map somebody attacking this would like — and there is no
|
||||||
|
// reason for it to be reachable from further away than the mesh.
|
||||||
|
listen := set.String("listen", "127.0.0.1:8080", "where to serve it")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
server := &http.Server{
|
||||||
|
Addr: *listen,
|
||||||
|
ReadHeaderTimeout: 10 * time.Second,
|
||||||
|
Handler: board(),
|
||||||
|
}
|
||||||
|
fmt.Printf("the board is on http://%s\n", *listen)
|
||||||
|
fmt.Printf(" it reads the mesh on every request and keeps nothing\n")
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-ctx.Done()
|
||||||
|
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
_ = server.Shutdown(closing)
|
||||||
|
}()
|
||||||
|
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// board is the handler, separate so a test can drive it without a listener.
|
||||||
|
func board() http.Handler {
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/" {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
asked, err := ask(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
// **Said, not blank.** A board that cannot reach the mesh and renders an empty page
|
||||||
|
// says "nothing is wrong" in the one situation where nobody can know that.
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
_ = page.Execute(w, view{Unreachable: err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
if err := page.Execute(w, asked); err != nil {
|
||||||
|
// The page is half-written by now; there is nothing useful left to say to the
|
||||||
|
// browser, and saying it here is what stops the failure being silent.
|
||||||
|
fmt.Printf("the board could not render: %v\n", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
// The same answers for something that is not a person, from the same read. A board and a
|
||||||
|
// script disagreeing about which machine is broken would be worse than either alone.
|
||||||
|
mux.HandleFunc("/mesh.json", func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
open, err := openStores(r.Context())
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
asked, err := theThreeQuestions(r.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write(append(body, '\n'))
|
||||||
|
})
|
||||||
|
return mux
|
||||||
|
}
|
||||||
|
|
||||||
|
// ask reads the mesh for one request.
|
||||||
|
func ask(ctx context.Context) (view, error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return view{}, err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return view{}, err
|
||||||
|
}
|
||||||
|
return viewOf(asked), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// view is what the page is given. Nothing is derived here that the reader could not derive.
|
||||||
|
type view struct {
|
||||||
|
Unreachable string
|
||||||
|
Machines int
|
||||||
|
Broken []brokenMachine
|
||||||
|
Quiet []quietMachine
|
||||||
|
Behind []staleModule
|
||||||
|
Waiting []waitingMachine
|
||||||
|
// Unresolved is every machine that cannot be worked out at all. Shown above everything else,
|
||||||
|
// because a machine here is in none of the other lists: nothing was computed for it, so it is
|
||||||
|
// not broken, not quiet and not behind — and a page without this said "all well" about a mesh
|
||||||
|
// where nothing could be sent anywhere.
|
||||||
|
Unresolved []blockedMachine
|
||||||
|
// Network is why the private network could not be computed, when it could not.
|
||||||
|
Network string
|
||||||
|
// Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the
|
||||||
|
// flip, so a node left adopted is shown rather than read as converged.
|
||||||
|
Adopted []string
|
||||||
|
At string
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockedMachine struct {
|
||||||
|
Node string
|
||||||
|
// Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
|
||||||
|
// a machine is usually blocked by more than one and fixing one of them changes nothing.
|
||||||
|
Said []string
|
||||||
|
}
|
||||||
|
|
||||||
|
type waitingMachine struct {
|
||||||
|
Node string
|
||||||
|
// Never told is not out of date: nobody has ever asked this machine to be anything. Same
|
||||||
|
// remedy, different situation, and the page says which.
|
||||||
|
Never bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type brokenMachine struct {
|
||||||
|
Node string
|
||||||
|
// Outcome is refused or failed, and stays distinct all the way to the page. **Refused means
|
||||||
|
// the machine is exactly as it was and what is wrong is in what was sent; failed means it is
|
||||||
|
// in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
||||||
|
// different places, so one word for both would send half the readers to the wrong one.
|
||||||
|
Outcome string
|
||||||
|
Said []string
|
||||||
|
When string
|
||||||
|
}
|
||||||
|
|
||||||
|
type quietMachine struct {
|
||||||
|
Node string
|
||||||
|
// Heard is "never" or how long ago. Never heard from is not the same as quiet for a while:
|
||||||
|
// one may be a machine that was never sent anything.
|
||||||
|
Heard string
|
||||||
|
}
|
||||||
|
|
||||||
|
type staleModule struct {
|
||||||
|
Module string
|
||||||
|
Holds string
|
||||||
|
Source string
|
||||||
|
Running []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func viewOf(asked answers) view {
|
||||||
|
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
||||||
|
Network: asked.network, Adopted: adoptedNodes(asked.nodes)}
|
||||||
|
var blocked []string
|
||||||
|
for name := range asked.refused {
|
||||||
|
blocked = append(blocked, name)
|
||||||
|
}
|
||||||
|
sort.Strings(blocked)
|
||||||
|
for _, name := range blocked {
|
||||||
|
one := blockedMachine{Node: name}
|
||||||
|
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
||||||
|
one.Said = append(one.Said, strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
out.Unresolved = append(out.Unresolved, one)
|
||||||
|
}
|
||||||
|
for _, d := range asked.wrong {
|
||||||
|
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
|
||||||
|
When: d.At.Local().Format("2006-01-02 15:04")}
|
||||||
|
if d.Refused != "" {
|
||||||
|
// The host's own words. It says exactly what it could not accept, and nothing
|
||||||
|
// written here would say it better.
|
||||||
|
one.Said = append(one.Said, firstLine(d.Refused))
|
||||||
|
}
|
||||||
|
for _, f := range d.Failed {
|
||||||
|
one.Said = append(one.Said, f.ID+": "+firstLine(f.Error))
|
||||||
|
}
|
||||||
|
out.Broken = append(out.Broken, one)
|
||||||
|
}
|
||||||
|
for _, n := range asked.quiet {
|
||||||
|
out.Quiet = append(out.Quiet, quietMachine{Node: n.Name, Heard: heardFrom(n)})
|
||||||
|
}
|
||||||
|
for _, m := range asked.waiting {
|
||||||
|
out.Waiting = append(out.Waiting, waitingMachine{Node: m.Node, Never: m.Never})
|
||||||
|
}
|
||||||
|
for module, on := range asked.behind {
|
||||||
|
from := asked.sources[module]
|
||||||
|
out.Behind = append(out.Behind, staleModule{
|
||||||
|
Module: module, Holds: short(from.BuiltFrom), Source: short(from.Head), Running: on,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// The page. Deliberately one file with no assets: a board that cannot render without fetching
|
||||||
|
// something is a board that is blank exactly when the mesh is unwell.
|
||||||
|
var page = template.Must(template.New("board").Parse(`<!doctype html>
|
||||||
|
<meta charset="utf-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<title>the mesh</title>
|
||||||
|
<style>
|
||||||
|
:root { color-scheme: light dark; }
|
||||||
|
body { font: 15px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; margin: 2rem auto;
|
||||||
|
max-width: 52rem; padding: 0 1rem; }
|
||||||
|
h1 { font-size: 1.1rem; font-weight: 600; margin: 0 0 1.5rem; }
|
||||||
|
h2 { font-size: 1rem; font-weight: 600; margin: 2rem 0 .5rem; }
|
||||||
|
.quiet { opacity: .65; }
|
||||||
|
.said { opacity: .8; padding-left: 1.5rem; }
|
||||||
|
.outcome { display: inline-block; min-width: 4.5rem; }
|
||||||
|
.refused { color: #b26b00; }
|
||||||
|
.failed { color: #c0392b; }
|
||||||
|
ul { list-style: none; padding: 0; margin: 0; }
|
||||||
|
li { padding: .15rem 0; }
|
||||||
|
footer { margin-top: 3rem; opacity: .6; font-size: .85rem; }
|
||||||
|
</style>
|
||||||
|
{{if .Unreachable}}
|
||||||
|
<h1>the mesh cannot be read</h1>
|
||||||
|
<p class="failed">{{.Unreachable}}</p>
|
||||||
|
<p class="quiet">This says nothing about whether the mesh is well — only that this page could not
|
||||||
|
find out.</p>
|
||||||
|
{{else}}
|
||||||
|
<h1>{{.Machines}} machine{{if ne .Machines 1}}s{{end}}</h1>
|
||||||
|
|
||||||
|
{{if .Unresolved}}
|
||||||
|
<h2>Can everything be worked out?</h2>
|
||||||
|
<ul>
|
||||||
|
{{range .Unresolved}}
|
||||||
|
<li><span class="outcome failed">blocked</span> <strong>{{.Node}}</strong>
|
||||||
|
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
||||||
|
</li>
|
||||||
|
{{end}}
|
||||||
|
</ul>
|
||||||
|
<p class="quiet">Nothing can be sent to a machine here, and it appears in none of the lists below:
|
||||||
|
nothing was computed for it, so there is nothing it can be behind.</p>
|
||||||
|
{{end}}
|
||||||
|
{{if .Network}}
|
||||||
|
<p class="failed">The private network could not be computed: {{.Network}}</p>
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
<h2>Is anything broken?</h2>
|
||||||
|
{{if .Broken}}
|
||||||
|
<ul>
|
||||||
|
{{range .Broken}}
|
||||||
|
<li>
|
||||||
|
<span class="outcome {{.Outcome}}">{{.Outcome}}</span>
|
||||||
|
<strong>{{.Node}}</strong> <span class="quiet">{{.When}}</span>
|
||||||
|
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
||||||
|
</li>
|
||||||
|
{{end}}
|
||||||
|
</ul>
|
||||||
|
{{else}}<p class="quiet">No. Every machine is doing what it was told.</p>{{end}}
|
||||||
|
|
||||||
|
<h2>Is anything not answering?</h2>
|
||||||
|
{{if .Quiet}}
|
||||||
|
<ul>{{range .Quiet}}<li><strong>{{.Node}}</strong> <span class="quiet">{{.Heard}}</span></li>{{end}}</ul>
|
||||||
|
<p class="quiet">Not heard from is not the same as tried and could not — a machine here may be
|
||||||
|
new, switched off, or unreachable.</p>
|
||||||
|
{{else}}<p class="quiet">No. Every machine has been heard from.</p>{{end}}
|
||||||
|
|
||||||
|
<h2>Is anything out of date?</h2>
|
||||||
|
{{if .Behind}}
|
||||||
|
<ul>
|
||||||
|
{{range .Behind}}
|
||||||
|
<li><strong>{{.Module}}</strong> holds {{.Holds}}, source has {{.Source}}
|
||||||
|
{{if .Running}}<div class="said">running on {{range $i, $n := .Running}}{{if $i}}, {{end}}{{$n}}{{end}}</div>
|
||||||
|
{{else}}<div class="said quiet">assigned to nothing</div>{{end}}
|
||||||
|
</li>
|
||||||
|
{{end}}
|
||||||
|
</ul>
|
||||||
|
{{else}}<p class="quiet">No. Every module is what its source last had.</p>{{end}}
|
||||||
|
|
||||||
|
{{if .Waiting}}
|
||||||
|
<ul>
|
||||||
|
{{range .Waiting}}
|
||||||
|
<li><strong>{{.Node}}</strong>
|
||||||
|
{{if .Never}}<span class="quiet">has never been sent anything</span>
|
||||||
|
{{else}}<span class="quiet">is not running what the mesh would send it</span>{{end}}
|
||||||
|
</li>
|
||||||
|
{{end}}
|
||||||
|
</ul>
|
||||||
|
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
||||||
|
Both are sent by <code>push --behind</code>.</p>
|
||||||
|
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
||||||
|
{{if .Adopted}}
|
||||||
|
<h2>Which machines are adopted?</h2>
|
||||||
|
<ul>{{range .Adopted}}<li><strong>{{.}}</strong> <span class="quiet">adopted — what was found on it is kept until each module is taken</span></li>{{end}}</ul>
|
||||||
|
{{end}}
|
||||||
|
{{end}}
|
||||||
|
|
||||||
|
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
||||||
|
`))
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Refused and failed stay distinct all the way to the page.
|
||||||
|
//
|
||||||
|
// **Refused means the machine is exactly as it was and what is wrong is in what was sent; failed
|
||||||
|
// means it is in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
||||||
|
// different places, so a page saying "error" for both sends half its readers to the wrong one.
|
||||||
|
func TestRefusedAndFailedReachThePageAsDifferentThings(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||||
|
wrong: []inventory.Doing{
|
||||||
|
{Node: "anchor", Outcome: "refused", Refused: "not a declaration this host speaks",
|
||||||
|
At: time.Now()},
|
||||||
|
{Node: "laptop", Outcome: "failed", At: time.Now(),
|
||||||
|
Failed: []inventory.FailedResource{{ID: "web.container", Error: "no such image"}}},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
// The rendered outcome, not the word anywhere on the page: both words appear in the
|
||||||
|
// stylesheet, so a plain Contains passes whatever the machine actually said. It did.
|
||||||
|
for _, want := range []string{`<span class="outcome refused">refused</span>`,
|
||||||
|
`<span class="outcome failed">failed</span>`} {
|
||||||
|
if !strings.Contains(rendered, want) {
|
||||||
|
t.Fatalf("the page does not distinguish the two — missing %s:\n%s", want, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And the host's own words, which say exactly what it could not accept.
|
||||||
|
for _, want := range []string{"not a declaration this host speaks", "web.container", "no such image"} {
|
||||||
|
if !strings.Contains(rendered, want) {
|
||||||
|
t.Fatalf("the page does not say %q, so a reader must go and ask:\n%s", want, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing wrong is said, not left blank. An empty page and a well mesh must not look alike.
|
||||||
|
func TestAWellMeshSaysSoRatherThanShowingNothing(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
||||||
|
for _, want := range []string{
|
||||||
|
"Every machine is doing what it was told",
|
||||||
|
"Every machine has been heard from",
|
||||||
|
"Every module is what its source last had",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(rendered, want) {
|
||||||
|
t.Fatalf("a well mesh does not say %q:\n%s", want, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A board that cannot reach the mesh must not render an empty page: that says "nothing is wrong"
|
||||||
|
// in the one situation where nobody can know it.
|
||||||
|
func TestABoardThatCannotReadTheMeshSaysSo(t *testing.T) {
|
||||||
|
rendered := render(t, view{Unreachable: "the store did not answer"})
|
||||||
|
if !strings.Contains(rendered, "the store did not answer") {
|
||||||
|
t.Fatalf("the reason is not on the page:\n%s", rendered)
|
||||||
|
}
|
||||||
|
if strings.Contains(rendered, "Every machine is doing what it was told") {
|
||||||
|
t.Fatal("a board that could not read the mesh reported that the mesh is well")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Quiet is not broken, and the page says which it is.
|
||||||
|
func TestQuietIsNotReportedAsBroken(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "laptop"}},
|
||||||
|
quiet: []inventory.Node{{Name: "laptop"}},
|
||||||
|
}))
|
||||||
|
if !strings.Contains(rendered, "not the same as tried and could not") {
|
||||||
|
t.Fatalf("the page does not separate quiet from broken:\n%s", rendered)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rendered, "Every machine is doing what it was told") {
|
||||||
|
t.Fatalf("a quiet machine was counted as broken:\n%s", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The page renders what a machine said, and a hostile string in it is not markup.
|
||||||
|
//
|
||||||
|
// What is on this page comes from machines, and a machine's own words are the whole reason the
|
||||||
|
// page is useful. They are also the one thing here that nobody in this repository wrote.
|
||||||
|
func TestWhatAMachineSaidIsNotMarkup(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "anchor"}},
|
||||||
|
wrong: []inventory.Doing{{Node: "anchor", Outcome: "refused",
|
||||||
|
Refused: `<script>alert("from the machine")</script>`, At: time.Now()}},
|
||||||
|
}))
|
||||||
|
if strings.Contains(rendered, "<script>alert") {
|
||||||
|
t.Fatalf("a machine's words were rendered as markup:\n%s", rendered)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rendered, "<script>") {
|
||||||
|
t.Fatalf("the words were not shown at all, so the reader cannot see what it said:\n%s", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module behind its source names the machines running the old one — the part with consequences.
|
||||||
|
func TestBeingBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "laptop"}},
|
||||||
|
behind: map[string][]string{"web": {"laptop"}},
|
||||||
|
sources: map[string]inventory.Source{"web": {BuiltFrom: "aaaaaaaaaa", Head: "bbbbbbbbbb"}},
|
||||||
|
}))
|
||||||
|
for _, want := range []string{"web", "aaaaaaaa", "bbbbbbbb", "running on laptop"} {
|
||||||
|
if !strings.Contains(rendered, want) {
|
||||||
|
t.Fatalf("the page does not say %q:\n%s", want, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func render(t *testing.T, v view) string {
|
||||||
|
t.Helper()
|
||||||
|
var out strings.Builder
|
||||||
|
if err := page.Execute(&out, v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return out.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// The board and the command must answer the same question the same way.
|
||||||
|
//
|
||||||
|
// A machine not running what the mesh would send it was added to `status` and would have been
|
||||||
|
// missing here — which is the one thing this page's design forbids: two answers to the same
|
||||||
|
// question, disagreeing, with a person in front of each.
|
||||||
|
func TestTheBoardSaysWhichMachinesHaveNotBeenSentWhatTheyShouldBe(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||||
|
waiting: []inventory.Machine{
|
||||||
|
{Node: "anchor"},
|
||||||
|
{Node: "laptop", Never: true},
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
if !strings.Contains(rendered, "is not running what the mesh would send it") {
|
||||||
|
t.Fatalf("the page does not say a machine is out of date:\n%s", rendered)
|
||||||
|
}
|
||||||
|
// Never told is a different situation with the same remedy, and the page keeps them apart.
|
||||||
|
if !strings.Contains(rendered, "has never been sent anything") {
|
||||||
|
t.Fatalf("the page does not separate never told from out of date:\n%s", rendered)
|
||||||
|
}
|
||||||
|
if !strings.Contains(rendered, "push --behind") {
|
||||||
|
t.Fatalf("the page does not say what sends them:\n%s", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And says so when there is nothing waiting, rather than leaving the question unanswered.
|
||||||
|
func TestAMeshWithNothingWaitingSaysSo(t *testing.T) {
|
||||||
|
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
||||||
|
if !strings.Contains(rendered, "Every machine is running what the mesh would send it") {
|
||||||
|
t.Fatalf("the page leaves the question unanswered:\n%s", rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,831 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// asking a build machine for a module, and what came back.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
// buildCommand builds a module from its source and records what came out.
|
||||||
|
//
|
||||||
|
// **Run where there is a container runtime**, which is why it is a command rather than something
|
||||||
|
// the control plane does on its own: building needs to run things on a machine, and what the
|
||||||
|
// control plane may send a machine is bounded by the declaration language. This is the shape the
|
||||||
|
// builder module will take when it is given work over the broker; today a person runs it, and the
|
||||||
|
// mesh records the result the same way either way.
|
||||||
|
// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the
|
||||||
|
// rebuild a changed base needs, which nothing else asks for: their sources did not move, and
|
||||||
|
// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too.
|
||||||
|
func buildOn(ctx context.Context, base string, wait time.Duration) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
held, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
against, err := open.inventory.BuiltAgainst(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var on []inventory.Entry
|
||||||
|
for _, e := range held {
|
||||||
|
if standsOnModule(e, base, against) {
|
||||||
|
on = append(on, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(on) == 0 {
|
||||||
|
fmt.Printf("nothing the mesh holds stands on %s\n", base)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
on = orderByBases(on, against)
|
||||||
|
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
|
||||||
|
var failed []string
|
||||||
|
for _, e := range on {
|
||||||
|
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||||
|
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
|
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||||
|
fmt.Printf(" %v\n", err)
|
||||||
|
failed = append(failed, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", "))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func buildCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||||
|
ref := set.String("ref", "", "the branch, tag or commit to build")
|
||||||
|
// A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's
|
||||||
|
// root, which is the ordinary case and why this is a flag rather than a second argument.
|
||||||
|
path := set.String("path", "", "the module's directory inside the repository")
|
||||||
|
wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
|
||||||
|
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
|
||||||
|
// Every module whose source has moved, rather than one named repository.
|
||||||
|
//
|
||||||
|
// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
|
||||||
|
// already knows which modules are behind their source, so making a person read that list and
|
||||||
|
// retype each repository is asking them to be the loop. Naming a repository and asking which
|
||||||
|
// ones need building are different requests, so they are not combined.
|
||||||
|
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
|
||||||
|
on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs")
|
||||||
|
// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
|
||||||
|
// the repository is external, cloned exactly as given — see source.go.
|
||||||
|
self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *on != "" {
|
||||||
|
if len(positionals) != 0 || *behind || *self {
|
||||||
|
return errors.New("build --on <module> names a base and nothing else")
|
||||||
|
}
|
||||||
|
return buildOn(ctx, *on, *wait)
|
||||||
|
}
|
||||||
|
|
||||||
|
if *behind {
|
||||||
|
if len(positionals) != 0 || *self {
|
||||||
|
return errors.New("build <repository> or build --behind, not both: one names a " +
|
||||||
|
"repository and the other asks which need building")
|
||||||
|
}
|
||||||
|
return buildBehind(ctx, *wait)
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on <module>")
|
||||||
|
}
|
||||||
|
source := buildSource{Repository: positionals[0]}
|
||||||
|
if *self {
|
||||||
|
if err := onASeat(source.Repository); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source.Seat = gitSeat
|
||||||
|
}
|
||||||
|
|
||||||
|
if *dryRun {
|
||||||
|
return buildAndShow(ctx, source, *path, *ref, *wait)
|
||||||
|
}
|
||||||
|
return buildOne(ctx, source, *path, *ref, *wait)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildFrom turns what a builder said into what the mesh keeps.
|
||||||
|
//
|
||||||
|
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||||
|
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||||
|
// reference and composes the store's address back in where a reference is used. `against` — what
|
||||||
|
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
|
||||||
|
// artifact and not the machine it was pulled from.
|
||||||
|
func buildFrom(result link.BuildResult) inventory.Build {
|
||||||
|
kept := inventory.Build{
|
||||||
|
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||||
|
Commit: result.Commit, On: result.On, Failed: result.Failed,
|
||||||
|
// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
|
||||||
|
// The catalogue turns the manifest into requires/provides edges and `against` into build
|
||||||
|
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||||
|
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||||
|
// rebuild the graph rather than a list of names.
|
||||||
|
Path: result.Path,
|
||||||
|
}
|
||||||
|
// When it was asked, which is what orders it against another build of the same module
|
||||||
|
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||||
|
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
||||||
|
kept.Asked = asked
|
||||||
|
}
|
||||||
|
for _, ref := range result.Against {
|
||||||
|
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||||
|
}
|
||||||
|
for _, r := range result.Read {
|
||||||
|
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||||
|
}
|
||||||
|
var announced []inventory.Artifact
|
||||||
|
for _, made := range result.Made {
|
||||||
|
announced = append(announced, inventory.Artifact{
|
||||||
|
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||||
|
})
|
||||||
|
kept.Made = append(kept.Made, inventory.Artifact{
|
||||||
|
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||||
|
// The module name comes from the manifest, which only exists when the build got that far.
|
||||||
|
if len(kept.Manifest) > 0 {
|
||||||
|
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||||
|
kept.Module = m.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return kept
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildsCommand says what has been built lately.
|
||||||
|
func buildsCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
||||||
|
limit := set.Int("n", 20, "how many to show")
|
||||||
|
logOf := set.String("log", "", "a build's id: print what the build machine said, line by line")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *logOf != "" {
|
||||||
|
return buildLog(ctx, *logOf)
|
||||||
|
}
|
||||||
|
module := ""
|
||||||
|
if len(positionals) == 1 {
|
||||||
|
module = positionals[0]
|
||||||
|
} else if len(positionals) > 1 {
|
||||||
|
return errors.New("builds [<module>] [-n N]")
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
builds, err := inv.Builds(ctx, module, *limit)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(builds) == 0 {
|
||||||
|
// Said rather than printed as nothing: an empty list and a failed read must never look
|
||||||
|
// the same, and getting here means the store answered.
|
||||||
|
if module != "" {
|
||||||
|
fmt.Printf("nothing has been built for %s\n", module)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Println("nothing has been built yet")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, b := range builds {
|
||||||
|
what := b.Module
|
||||||
|
if what == "" {
|
||||||
|
// It failed before knowing what it was building, which is most of the interesting
|
||||||
|
// failures. The repository is what a person has to go and look at.
|
||||||
|
what = "?"
|
||||||
|
}
|
||||||
|
outcome := "built " + short(b.Commit)
|
||||||
|
if !b.Worked() {
|
||||||
|
outcome = "failed"
|
||||||
|
}
|
||||||
|
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
||||||
|
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||||
|
fmt.Printf(" %s", b.Repository)
|
||||||
|
if b.Ref != "" {
|
||||||
|
fmt.Printf(" at %s", b.Ref)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
for _, made := range b.Made {
|
||||||
|
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
||||||
|
}
|
||||||
|
if !b.Worked() {
|
||||||
|
// The builder's own first line. The whole failure is often a build log, and printing
|
||||||
|
// it here would bury every other row.
|
||||||
|
fmt.Printf(" %s\n", firstLine(b.Failed))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// builderCommand issues a build machine its own broker credential.
|
||||||
|
//
|
||||||
|
// **A build machine is not a node**, and giving it a node's account would let it read another
|
||||||
|
// machine's declarations. This is narrower and different: read the build queue, write the
|
||||||
|
// exchange and an asker's reply queue, and nothing else.
|
||||||
|
//
|
||||||
|
// Issued rather than assumed, because until this the builder used whatever credential it was
|
||||||
|
// handed — which in practice meant the broker's own administrative one. A program documented as
|
||||||
|
// holding its own credential and given somebody else's is worse than one with no story at all.
|
||||||
|
func builderCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("builder issue", flag.ContinueOnError)
|
||||||
|
// Which machine will use it. Given, the credential is delivered by the mesh rather than
|
||||||
|
// printed for somebody to carry — which is the difference between the builder being a module
|
||||||
|
// and being a program somebody configures.
|
||||||
|
forNode := set.String("node", "",
|
||||||
|
"the machine that will run it, so the mesh delivers the credential instead of printing it")
|
||||||
|
module := set.String("module", "builder", "the module on that machine that will read it")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 2 || positionals[0] != "issue" {
|
||||||
|
return errors.New("builder issue <name> [--node <machine>]")
|
||||||
|
}
|
||||||
|
name := positionals[1]
|
||||||
|
|
||||||
|
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
|
||||||
|
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
|
||||||
|
// broker secret, usable at the next push — the same act `module issue` performs, and the same
|
||||||
|
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m, known := shelf[*module]
|
||||||
|
if !known {
|
||||||
|
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
|
||||||
|
}
|
||||||
|
fmt.Printf("build machine %s: ", name)
|
||||||
|
node := *forNode
|
||||||
|
if node == "" {
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == *module && len(e.On) > 0 {
|
||||||
|
node = e.On[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if node == "" {
|
||||||
|
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return issueOnTheNewBus(ctx, inv, m, node, address)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildBehind builds every module the mesh holds older than its source has.
|
||||||
|
//
|
||||||
|
// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
|
||||||
|
// records where each module came from and what its source last had; until this, a person read
|
||||||
|
// that list and retyped each repository — which is a person being the loop, and the thing a
|
||||||
|
// pipeline was doing before it was taken away.
|
||||||
|
//
|
||||||
|
// Each is built and recorded on its own. **One failing does not stop the others**, for the same
|
||||||
|
// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
|
||||||
|
// repository holds back nine good ones is a mesh where nobody dares add the tenth.
|
||||||
|
func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
held, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var stale []inventory.Entry
|
||||||
|
for _, e := range held {
|
||||||
|
if !e.Source.Current() {
|
||||||
|
stale = append(stale, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(stale) == 0 {
|
||||||
|
// Said rather than doing nothing quietly: "nothing needed building" and "this did not
|
||||||
|
// run" must never look the same.
|
||||||
|
fmt.Println("every module the mesh holds is what its source last had")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("%d module(s) behind their source:\n", len(stale))
|
||||||
|
for _, e := range stale {
|
||||||
|
fmt.Printf(" %s %s < %s\n",
|
||||||
|
e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
// Bases first: a module built before the module it stands on is built against the old one
|
||||||
|
// and reports success (novox/hq 04-ISSUES/131).
|
||||||
|
against, err := inv.BuiltAgainst(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
stale = orderByBases(stale, against)
|
||||||
|
|
||||||
|
var failed []string
|
||||||
|
for _, e := range stale {
|
||||||
|
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||||
|
// Its own recorded ref, not its head commit: a module tracking a branch should be built
|
||||||
|
// from that branch, and pinning to the commit the mesh happened to notice would quietly
|
||||||
|
// turn a tracked branch into a pin.
|
||||||
|
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
|
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||||
|
fmt.Printf(" %v\n", err)
|
||||||
|
failed = append(failed, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(failed) > 0 {
|
||||||
|
return fmt.Errorf("%d of %d could not be built: %s",
|
||||||
|
len(failed), len(stale), strings.Join(failed, ", "))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
|
||||||
|
len(stale))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildOne asks a build machine for one repository and records everything that came back.
|
||||||
|
//
|
||||||
|
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||||
|
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||||
|
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||||
|
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||||
|
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||||
|
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||||
|
dryRun bool) (string, error) {
|
||||||
|
if dryRun && wait != 0 {
|
||||||
|
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||||
|
}
|
||||||
|
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||||
|
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||||
|
repository, err := cloneFrom(ctx, source)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
ident, err := openIdentity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer ident.Close()
|
||||||
|
|
||||||
|
server, err := connectLink(ctx, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
||||||
|
// module can be in flight, and the second answer is not the first one's.
|
||||||
|
request := link.BuildRequest{
|
||||||
|
ID: link.NewBuildID(time.Now()),
|
||||||
|
Repository: repository,
|
||||||
|
Path: path,
|
||||||
|
Ref: ref,
|
||||||
|
Held: heldBy(ctx),
|
||||||
|
Seats: seatBases(ctx),
|
||||||
|
DryRun: dryRun,
|
||||||
|
}
|
||||||
|
fmt.Printf("asked for %s", source)
|
||||||
|
if source.Seat != "" {
|
||||||
|
fmt.Printf(" (%s)", repository)
|
||||||
|
}
|
||||||
|
// The id is how a person follows this build while it runs: `builds --log <id>`.
|
||||||
|
fmt.Printf(" as %s", request.ID)
|
||||||
|
if path != "" {
|
||||||
|
fmt.Printf(" at %s", path)
|
||||||
|
}
|
||||||
|
if ref != "" {
|
||||||
|
fmt.Printf(" on %s", ref)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
ask, err := askOverOn(seat)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer ask.Close()
|
||||||
|
fmt.Printf(" of %s\n", seat)
|
||||||
|
|
||||||
|
if wait == 0 {
|
||||||
|
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||||
|
// controller takes it in — records the build, registers the module — whether or not anybody
|
||||||
|
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||||
|
// follows the build by its id instead.
|
||||||
|
if err := ask.Ask(ctx, request); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if dryRun {
|
||||||
|
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||||
|
"its outcome is not taken in\n", request.ID)
|
||||||
|
return request.ID, nil
|
||||||
|
}
|
||||||
|
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||||
|
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||||
|
return request.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := ask.Submit(ctx, request, wait)
|
||||||
|
if err != nil {
|
||||||
|
return request.ID, err
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return request.ID, err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||||
|
if err != nil {
|
||||||
|
return request.ID, err
|
||||||
|
}
|
||||||
|
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||||
|
for _, made := range kept.Made {
|
||||||
|
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n%s %s, built on %s from %s\n",
|
||||||
|
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
|
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||||
|
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||||
|
return request.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||||
|
// result on at once (novox/hq issue 126, ADR 0163): a person choreographing a data move must
|
||||||
|
// know which module will not wait for them.
|
||||||
|
func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module string) {
|
||||||
|
if u, err := inv.UpgradeOf(ctx, module); err == nil && u.RollOut {
|
||||||
|
how := "one machine at a time"
|
||||||
|
if u.Together {
|
||||||
|
how = "every machine at once"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s rolls out on build: the machines running it are sent this now, %s — "+
|
||||||
|
"`upgrade %s record` first if something must move before it does\n", module, how, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
|
||||||
|
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
|
||||||
|
// result reaches the catalogue whether or not the asker was still listening.
|
||||||
|
//
|
||||||
|
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
|
||||||
|
// nobody asked for, and the difference is the whole of whether somebody should be looking at
|
||||||
|
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
|
||||||
|
// would be a second thing to disagree about what a manifest is — and registered with where it came
|
||||||
|
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
|
||||||
|
// which the request carried and the outcome echoes. A definition naming an installation is refused
|
||||||
|
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
|
||||||
|
//
|
||||||
|
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
|
||||||
|
// written with the same values.
|
||||||
|
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||||
|
catalogue.Manifest, inventory.Build, error) {
|
||||||
|
kept := buildFrom(result)
|
||||||
|
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||||
|
return catalogue.Manifest{}, kept, err
|
||||||
|
}
|
||||||
|
if result.Failed != "" {
|
||||||
|
// The builder's own words. Wrapping them in something about the control plane would put
|
||||||
|
// two explanations between a person and a build log.
|
||||||
|
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||||
|
result.On, result.Repository, result.Failed)
|
||||||
|
}
|
||||||
|
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||||
|
result.On, result.Repository, err)
|
||||||
|
}
|
||||||
|
recorded := inventory.Source{
|
||||||
|
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||||
|
BuiltFrom: result.Commit, Head: result.Commit,
|
||||||
|
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||||
|
Against: kept.Against,
|
||||||
|
// When it was asked, so an older request heard later does not replace a newer one
|
||||||
|
// (novox/hq 04-ISSUES/219).
|
||||||
|
Asked: kept.Asked,
|
||||||
|
}
|
||||||
|
if result.Source != nil && result.Source.Seat != "" {
|
||||||
|
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||||
|
}
|
||||||
|
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
||||||
|
// the commit is built and recorded as what it was built from, and the module keeps following
|
||||||
|
// what it followed before — the repository's default branch for one new to the catalogue.
|
||||||
|
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
||||||
|
recorded.Ref = ""
|
||||||
|
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||||
|
recorded.Ref = followedBranch(was.Ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := namesNoInstallation(manifest); err != nil {
|
||||||
|
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||||
|
result.On, result.Repository, short(result.Commit), err)
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||||
|
if errors.Is(err, inventory.ErrSuperseded) {
|
||||||
|
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||||
|
result.On, manifest.Module, short(result.Commit), err)
|
||||||
|
}
|
||||||
|
return manifest, kept, err
|
||||||
|
}
|
||||||
|
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||||
|
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||||
|
// worked and the module is registered.
|
||||||
|
collect(ctx, inv)
|
||||||
|
return manifest, kept, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildAndShow builds and prints the manifest without recording anything.
|
||||||
|
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||||
|
repository, err := cloneFrom(ctx, source)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ident, err := openIdentity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ident.Close()
|
||||||
|
server, err := connectLink(ctx, nil, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ask.Close()
|
||||||
|
|
||||||
|
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||||
|
ID: link.NewBuildID(time.Now()),
|
||||||
|
Repository: repository, Path: path, Ref: ref,
|
||||||
|
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||||
|
DryRun: true,
|
||||||
|
}, wait)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if result.Failed != "" {
|
||||||
|
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||||
|
}
|
||||||
|
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||||
|
result.On, result.Repository, err)
|
||||||
|
}
|
||||||
|
body, err := json.MarshalIndent(manifest, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// answers is what the three questions came back with, read once.
|
||||||
|
type answers struct {
|
||||||
|
wrong []inventory.Doing
|
||||||
|
nodes []inventory.Node
|
||||||
|
quiet []inventory.Node
|
||||||
|
behind map[string][]string
|
||||||
|
sources map[string]inventory.Source
|
||||||
|
// waiting is every machine not running what the mesh would send it.
|
||||||
|
waiting []inventory.Machine
|
||||||
|
// reported is every machine's last word beside when it was last sent a declaration — the
|
||||||
|
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||||
|
// recorded at send, not at apply).
|
||||||
|
reported []inventory.Reported
|
||||||
|
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||||
|
plans []inventory.Plan
|
||||||
|
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||||
|
paused pauseView
|
||||||
|
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||||
|
// other answer here: those are about a machine that was told something, and this is about one
|
||||||
|
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||||
|
// to compare against, so without this a wholly blocked mesh reads as a well one.
|
||||||
|
refused map[string]string
|
||||||
|
// network is why the private network could not be computed, when it could not. Almost always
|
||||||
|
// a consequence of the refusals above: a node that does not resolve is not on the network, and
|
||||||
|
// a mesh whose hub is that node has no hub.
|
||||||
|
network string
|
||||||
|
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||||
|
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
|
||||||
|
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
|
||||||
|
filtered map[string]inventory.Filtering
|
||||||
|
// untaken is, per machine, each assigned module whose resources the machine is holding as it
|
||||||
|
// found them, and how many — a module that was assigned, sent, and is running none of what it
|
||||||
|
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
|
||||||
|
//
|
||||||
|
// **Its absence cost an outage.** The module was assigned, the push reported success, this
|
||||||
|
// command said the machine was doing everything it was told, and the module's three containers
|
||||||
|
// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
|
||||||
|
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||||
|
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||||
|
untaken map[string]map[string]int
|
||||||
|
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
|
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||||
|
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||||
|
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||||
|
unheld []catalogue.Unheld
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||||
|
//
|
||||||
|
// **A failure here is not a failure to build.** A module that names no base does not need this at
|
||||||
|
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
||||||
|
// than a build command refusing to start because a query did not run. So the store not opening is
|
||||||
|
// reported and the build goes ahead without it.
|
||||||
|
//
|
||||||
|
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||||
|
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||||
|
func heldBy(ctx context.Context) map[string]string {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||||
|
"base will be told that base is missing: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
held, err := open.inventory.Held(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
||||||
|
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
if address == "" {
|
||||||
|
return held
|
||||||
|
}
|
||||||
|
routed := make(map[string]string, len(held))
|
||||||
|
for repository, reference := range held {
|
||||||
|
routed[repository] = catalogue.Rerouted(reference, address)
|
||||||
|
}
|
||||||
|
return routed
|
||||||
|
}
|
||||||
|
|
||||||
|
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
|
||||||
|
//
|
||||||
|
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||||
|
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||||
|
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||||
|
func askOverOn(seat string) (link.Builders, error) {
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return link.BuildsOverNATSOn(address, seat)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||||
|
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||||
|
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||||
|
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||||
|
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||||
|
func buildSeatHeld(ctx context.Context) string {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||||
|
link.TheBuildMachine, err)
|
||||||
|
return link.TheBuildMachine
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||||
|
link.TheBuildMachine, err)
|
||||||
|
return link.TheBuildMachine
|
||||||
|
}
|
||||||
|
return buildSeatAmong(entries)
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||||
|
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||||
|
// the current role, which is where every ask goes once the handover is done.
|
||||||
|
func buildSeatAmong(entries []inventory.Entry) string {
|
||||||
|
heldBefore := false
|
||||||
|
for _, e := range entries {
|
||||||
|
if len(e.On) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||||
|
return link.TheBuildMachine
|
||||||
|
}
|
||||||
|
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||||
|
heldBefore = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if heldBefore {
|
||||||
|
return link.TheBuildMachineBefore
|
||||||
|
}
|
||||||
|
return link.TheBuildMachine
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||||
|
//
|
||||||
|
// **From the stream, not from a record** (novox/hq ADR 0157). A build's lines are the role's own
|
||||||
|
// events under the build's id, retained with every other event; the mesh keeps no second copy. Read
|
||||||
|
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||||
|
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||||
|
func buildLog(ctx context.Context, id string) error {
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
|
||||||
|
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||||
|
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||||
|
lines := link.BuildLogOf("*", id)
|
||||||
|
sub, err := js.Context().PullSubscribe(lines, "",
|
||||||
|
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
|
||||||
|
printed := 0
|
||||||
|
for {
|
||||||
|
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||||
|
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return fmt.Errorf("reading a build's log: %w", err)
|
||||||
|
}
|
||||||
|
for _, msg := range batch {
|
||||||
|
var line link.BuildLine
|
||||||
|
if err := json.Unmarshal(msg.Data, &line); err != nil {
|
||||||
|
fmt.Printf(" ? %s\n", string(msg.Data))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
at := line.At
|
||||||
|
if t, err := time.Parse(time.RFC3339Nano, line.At); err == nil {
|
||||||
|
at = t.Local().Format("15:04:05")
|
||||||
|
}
|
||||||
|
fmt.Printf("%s %4d [%s] %s\n", at, line.Seq, line.Step, line.Message)
|
||||||
|
printed++
|
||||||
|
}
|
||||||
|
if len(batch) < 200 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if printed == 0 {
|
||||||
|
fmt.Printf("nothing on the bus for build %s: no build by that id in the last week, or a build "+
|
||||||
|
"machine older than this that said nothing while building\n", id)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||||
|
return inventory.Entry{
|
||||||
|
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||||
|
On: on,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||||
|
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||||
|
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||||
|
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||||
|
onlyTheBuilder := []inventory.Entry{
|
||||||
|
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||||
|
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||||
|
}
|
||||||
|
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||||
|
t.Errorf("with only the builder assigned, asked %q", got)
|
||||||
|
}
|
||||||
|
bothHeld := []inventory.Entry{
|
||||||
|
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||||
|
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||||
|
}
|
||||||
|
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||||
|
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||||
|
}
|
||||||
|
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||||
|
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||||
|
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||||
|
}
|
||||||
|
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||||
|
t.Errorf("an empty catalogue asks %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build's outcome is taken in the same way whoever hears it (novox/hq issue 176): recorded, and
|
||||||
|
// the module registered with its source as the seat and path when the request said so — never the
|
||||||
|
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
||||||
|
// and said.
|
||||||
|
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||||
|
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||||
|
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||||
|
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Module != "shop" {
|
||||||
|
t.Fatalf("registered %q", m.Module)
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, held := shelf["shop"]; !held {
|
||||||
|
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||||
|
}
|
||||||
|
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||||
|
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||||
|
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||||
|
}
|
||||||
|
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||||
|
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||||
|
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||||
|
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||||
|
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||||
|
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||||
|
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||||
|
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||||
|
}
|
||||||
|
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||||
|
t.Fatal("the refused module was registered anyway")
|
||||||
|
}
|
||||||
|
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||||
|
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||||
|
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||||
|
// module keeps following the branch it followed — a new one, the default branch.
|
||||||
|
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||||
|
result := func(id, ref, commit string) link.BuildResult {
|
||||||
|
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||||
|
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||||
|
}
|
||||||
|
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||||
|
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||||
|
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||||
|
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||||
|
r.Manifest, r.Path = other, "modules/letta"
|
||||||
|
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||||
|
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||||
|
// so a push sends what the newer request built.
|
||||||
|
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||||
|
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||||
|
result := func(asked time.Time, image string) link.BuildResult {
|
||||||
|
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||||
|
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||||
|
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||||
|
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||||
|
}
|
||||||
|
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||||
|
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||||
|
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||||
|
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||||
|
}
|
||||||
|
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||||
|
t.Errorf("the late build was not recorded: %v", builds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||||
|
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
||||||
|
t.Errorf("read back %v %v; want %v", got, ok, at)
|
||||||
|
}
|
||||||
|
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
||||||
|
t.Errorf("the incident's id reads as %v %v", got, ok)
|
||||||
|
}
|
||||||
|
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
||||||
|
if _, ok := link.BuildAskedAt(id); ok {
|
||||||
|
t.Errorf("%q read as a request time", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/rsa"
|
||||||
|
"crypto/x509"
|
||||||
|
"crypto/x509/pkix"
|
||||||
|
"encoding/pem"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math/big"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The bus's own certificate, made by the mesh rather than borrowed from an image.
|
||||||
|
//
|
||||||
|
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
|
||||||
|
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
|
||||||
|
// image, which worked while the broker was one that happened to carry it and stopped the day the
|
||||||
|
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
|
||||||
|
// raised. Substituting another image the bundle names does not help — none of them carry it
|
||||||
|
// either.
|
||||||
|
//
|
||||||
|
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
|
||||||
|
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
|
||||||
|
// image it writes into but a mounted directory.
|
||||||
|
//
|
||||||
|
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
|
||||||
|
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
|
||||||
|
// this moment in a bootstrap there is no mesh to ask one of.
|
||||||
|
//
|
||||||
|
// Idempotent, because the step is applied again on every reconcile and a second certificate would
|
||||||
|
// be one the hosts that pinned the first no longer believe.
|
||||||
|
|
||||||
|
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
|
||||||
|
// loopback address the machine's own foundation dials.
|
||||||
|
var busCertificateNames = []string{"mesh-broker"}
|
||||||
|
|
||||||
|
const busCertificateLife = 10 * 365 * 24 * time.Hour
|
||||||
|
|
||||||
|
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
|
||||||
|
//
|
||||||
|
// broker certificate --into /tls make it if it is not there
|
||||||
|
// broker certificate --check --into /tls exit non-zero unless a usable pair is
|
||||||
|
func busCertificate(args []string) error {
|
||||||
|
into, check := "", false
|
||||||
|
for i := 0; i < len(args); i++ {
|
||||||
|
switch args[i] {
|
||||||
|
case "--check":
|
||||||
|
check = true
|
||||||
|
case "--into":
|
||||||
|
if i+1 >= len(args) {
|
||||||
|
return errors.New("--into needs a directory")
|
||||||
|
}
|
||||||
|
into = args[i+1]
|
||||||
|
i++
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if into == "" {
|
||||||
|
return errors.New("broker certificate [--check] --into <directory>")
|
||||||
|
}
|
||||||
|
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
|
||||||
|
|
||||||
|
if usable, err := busCertificateUsable(crt, key); err != nil {
|
||||||
|
return err
|
||||||
|
} else if usable {
|
||||||
|
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if check {
|
||||||
|
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
|
||||||
|
// this and a false answer is what makes the step run.
|
||||||
|
return fmt.Errorf("no usable certificate and key at %s", into)
|
||||||
|
}
|
||||||
|
return writeBusCertificate(crt, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// busCertificateUsable says whether a certificate and its key are both there and parse.
|
||||||
|
//
|
||||||
|
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
|
||||||
|
// between the two files leaves behind, and a step that treated it as done would hand the server a
|
||||||
|
// certificate with no key and report success.
|
||||||
|
func busCertificateUsable(crt, key string) (bool, error) {
|
||||||
|
certPEM, err := os.ReadFile(crt)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
keyPEM, err := os.ReadFile(key)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
|
||||||
|
block, _ := pem.Decode(certPEM)
|
||||||
|
if block == nil || block.Type != "CERTIFICATE" {
|
||||||
|
return nil, errors.New("not a certificate")
|
||||||
|
}
|
||||||
|
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
keyBlock, _ := pem.Decode(keyPEM)
|
||||||
|
if keyBlock == nil {
|
||||||
|
return nil, errors.New("not a key")
|
||||||
|
}
|
||||||
|
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
|
||||||
|
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return certificate, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeBusCertificate(crt, key string) error {
|
||||||
|
private, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
template := &x509.Certificate{
|
||||||
|
SerialNumber: serial,
|
||||||
|
Subject: pkix.Name{CommonName: busCertificateNames[0]},
|
||||||
|
DNSNames: busCertificateNames,
|
||||||
|
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||||
|
NotBefore: time.Now().Add(-time.Hour),
|
||||||
|
NotAfter: time.Now().Add(busCertificateLife),
|
||||||
|
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||||
|
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||||
|
BasicConstraintsValid: true,
|
||||||
|
}
|
||||||
|
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The key first, and only then the certificate**, so the pair a reader finds is never a
|
||||||
|
// certificate whose key has not been written yet — the one order in which an interruption
|
||||||
|
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
|
||||||
|
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
|
||||||
|
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// busAccounts writes the mesh's composed user list to a file.
|
||||||
|
//
|
||||||
|
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
|
||||||
|
// the list reaches the machine running the bus as a resource of the module that holds it — which
|
||||||
|
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
|
||||||
|
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
|
||||||
|
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
|
||||||
|
// file over from its first push.
|
||||||
|
//
|
||||||
|
// The same composition, not a second one: this asks the store for the same records and renders them
|
||||||
|
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
|
||||||
|
// second statement of who may say what, able to disagree with the first.
|
||||||
|
//
|
||||||
|
// **It writes to standard output unless told a file**, and that is the point: the control plane
|
||||||
|
// composes and says what it composed, and whoever is raising the machine puts it where that
|
||||||
|
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
|
||||||
|
// know where that is and how to make the server re-read it — which is the module's knowledge, and
|
||||||
|
// the module is what takes this over on the first push.
|
||||||
|
//
|
||||||
|
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
|
||||||
|
func busAccounts(ctx context.Context, args []string) error {
|
||||||
|
into := ""
|
||||||
|
for i := 0; i < len(args); i++ {
|
||||||
|
switch args[i] {
|
||||||
|
case "--into":
|
||||||
|
if i+1 >= len(args) {
|
||||||
|
return errors.New("--into needs a file")
|
||||||
|
}
|
||||||
|
into = args[i+1]
|
||||||
|
i++
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
records, err := open.inventory.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
kept, err := open.inventory.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hashes := make(map[string]string, len(kept))
|
||||||
|
for name, u := range kept {
|
||||||
|
hashes[name] = u.PasswordHash
|
||||||
|
}
|
||||||
|
filled, missing := broker.WithPasswords(users, hashes)
|
||||||
|
if len(missing) > 0 {
|
||||||
|
// To standard error, always: the composed file may be going to standard output, and a
|
||||||
|
// remark in the middle of it is a configuration the server refuses to parse.
|
||||||
|
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
|
||||||
|
len(missing), strings.Join(missing, ", "))
|
||||||
|
}
|
||||||
|
if len(filled) == 0 {
|
||||||
|
return errors.New("not one user has a credential, so this list would refuse every " +
|
||||||
|
"connection in the mesh")
|
||||||
|
}
|
||||||
|
accounts, err := broker.ComposeAccounts(filled)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if into == "" {
|
||||||
|
fmt.Print(accounts)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/tls"
|
||||||
|
"crypto/x509"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
|
||||||
|
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
|
||||||
|
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
|
||||||
|
|
||||||
|
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
|
||||||
|
into := t.TempDir()
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatalf("the bus could not be given a certificate: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The check a cheaper test would not make. The key was present and valid and the server could
|
||||||
|
// not start, once, because nothing loaded the pair the way a server loads it
|
||||||
|
// (novox/hq 04-ISSUES/014).
|
||||||
|
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a TLS server cannot load what was written: %v", err)
|
||||||
|
}
|
||||||
|
leaf := pair.Leaf
|
||||||
|
if leaf == nil {
|
||||||
|
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
|
||||||
|
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
|
||||||
|
}
|
||||||
|
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
|
||||||
|
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The key is not readable by anything else on the machine; the certificate is public and is.
|
||||||
|
key, err := os.Stat(filepath.Join(into, "tls.key"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if key.Mode().Perm() != 0o600 {
|
||||||
|
t.Errorf("the key is %v", key.Mode().Perm())
|
||||||
|
}
|
||||||
|
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if crt.Mode().Perm() != 0o644 {
|
||||||
|
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
|
||||||
|
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
|
||||||
|
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
|
||||||
|
into := t.TempDir()
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(first) != string(again) {
|
||||||
|
t.Fatal("running it twice replaced the certificate every host had pinned")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
|
||||||
|
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
|
||||||
|
into := t.TempDir()
|
||||||
|
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||||
|
t.Fatal("an empty directory reported a usable certificate")
|
||||||
|
}
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
|
||||||
|
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
|
||||||
|
// called it done would hand the server a certificate with no key and report success.
|
||||||
|
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
|
||||||
|
into := t.TempDir()
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||||
|
t.Fatal("a certificate with no key passed the check")
|
||||||
|
}
|
||||||
|
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
|
||||||
|
t.Fatalf("it did not replace the unusable pair: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhereToWriteIsRequired(t *testing.T) {
|
||||||
|
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
|
||||||
|
t.Fatalf("it did not ask where to write: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||||
|
//
|
||||||
|
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
|
||||||
|
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
|
||||||
|
// over exactly the manifests given. Somebody describing their own application in their own
|
||||||
|
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
|
||||||
|
// the registration or, later, when a machine applies something that resolved and should not have.
|
||||||
|
//
|
||||||
|
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
|
||||||
|
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
|
||||||
|
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||||
|
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||||
|
// refused what it could not see would teach people to ignore it.
|
||||||
|
func moduleCheck(paths []string, out io.Writer) error {
|
||||||
|
if len(paths) == 0 {
|
||||||
|
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||||
|
"manifest of a repository together so the rules between them are checked too")
|
||||||
|
}
|
||||||
|
shelf := catalogue.Shelf{}
|
||||||
|
faulted := map[string]bool{}
|
||||||
|
failed := 0
|
||||||
|
for _, path := range paths {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||||
|
failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||||
|
failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if first, twice := shelf[m.Module]; twice {
|
||||||
|
_ = first
|
||||||
|
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
|
||||||
|
failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||||
|
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||||
|
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||||
|
for _, p := range named {
|
||||||
|
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||||
|
}
|
||||||
|
failed += len(named)
|
||||||
|
faulted[m.Module] = true
|
||||||
|
}
|
||||||
|
shelf[m.Module] = m
|
||||||
|
}
|
||||||
|
|
||||||
|
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||||
|
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||||
|
// has already been said and would be said again here in a worse form.
|
||||||
|
problems := catalogue.CatalogueProblems(shelf)
|
||||||
|
sort.Strings(problems)
|
||||||
|
for _, p := range problems {
|
||||||
|
fmt.Fprintln(out, p)
|
||||||
|
}
|
||||||
|
failed += len(problems)
|
||||||
|
|
||||||
|
var names []string
|
||||||
|
for name := range shelf {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, name := range names {
|
||||||
|
m := shelf[name]
|
||||||
|
if faulted[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%s: ok", name)
|
||||||
|
if n := len(m.Tools); n > 0 {
|
||||||
|
fmt.Fprintf(out, ", %d tool(s)", n)
|
||||||
|
}
|
||||||
|
if len(m.Invokes) > 0 {
|
||||||
|
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||||
|
}
|
||||||
|
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||||
|
if len(m.State) > 0 {
|
||||||
|
kept := make([]string, 0, len(m.State))
|
||||||
|
for _, s := range m.State {
|
||||||
|
kept = append(kept, s.Name)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||||
|
}
|
||||||
|
if len(m.Reads) > 0 {
|
||||||
|
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||||
|
}
|
||||||
|
fmt.Fprintln(out)
|
||||||
|
}
|
||||||
|
if failed > 0 {
|
||||||
|
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||||
|
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||||
|
"module not given here reads as unknown\n", len(paths))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinInvokes(invokes []string) string {
|
||||||
|
if len(invokes) == 1 && invokes[0] == "*" {
|
||||||
|
return "every tool"
|
||||||
|
}
|
||||||
|
s := ""
|
||||||
|
for i, t := range invokes {
|
||||||
|
if i > 0 {
|
||||||
|
s += ", "
|
||||||
|
}
|
||||||
|
s += t
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
|
||||||
|
func manifestsUnder(dir string) ([]string, error) {
|
||||||
|
var found []string
|
||||||
|
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
|
||||||
|
return filepath.SkipDir
|
||||||
|
}
|
||||||
|
if !d.IsDir() && d.Name() == "module.json" {
|
||||||
|
found = append(found, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
sort.Strings(found)
|
||||||
|
return found, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||||
|
// with a known fault is named, and one without passes, with no store opened.
|
||||||
|
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
good := filepath.Join(dir, "good.json")
|
||||||
|
bad := filepath.Join(dir, "bad.json")
|
||||||
|
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
|
||||||
|
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheck([]string{good}, &out); err != nil {
|
||||||
|
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
|
||||||
|
t.Fatalf("the report does not say what it checked:\n%s", out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
out.Reset()
|
||||||
|
err := moduleCheck([]string{good, bad}, &out)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a manifest invoking a module and no tool passed")
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
|
||||||
|
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The rules between manifests run over what was given together: a seat two modules declare is
|
||||||
|
// refused, which no single-manifest check can see.
|
||||||
|
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
a := filepath.Join(dir, "a.json")
|
||||||
|
b := filepath.Join(dir, "b.json")
|
||||||
|
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||||
|
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheck([]string{a, b}, &out); err == nil {
|
||||||
|
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(out.String(), "a seat name means one protocol") {
|
||||||
|
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||||
|
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||||
|
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||||
|
if _, err := os.Stat(root); err != nil {
|
||||||
|
t.Skipf("catalogue sibling not present: %v", err)
|
||||||
|
}
|
||||||
|
paths, err := manifestsUnder(root)
|
||||||
|
if err != nil || len(paths) == 0 {
|
||||||
|
t.Fatalf("no manifests under %s: %v", root, err)
|
||||||
|
}
|
||||||
|
var out bytes.Buffer
|
||||||
|
if err := moduleCheck(paths, &out); err != nil {
|
||||||
|
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
||||||
|
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
|
||||||
|
// ways in — `module add` and a build's result — go through this, and a name declared on
|
||||||
|
// purpose passes with its reason.
|
||||||
|
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
|
||||||
|
{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||||
|
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
|
||||||
|
}}
|
||||||
|
err := namesNoInstallation(named)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
|
||||||
|
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
|
||||||
|
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
|
||||||
|
}
|
||||||
|
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
|
||||||
|
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
|
||||||
|
catalogue.NamesOnPurpose: map[string]any{
|
||||||
|
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
|
||||||
|
}}
|
||||||
|
if err := namesNoInstallation(meant); err != nil {
|
||||||
|
t.Fatalf("a name declared on purpose passes; got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,175 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||||
|
//
|
||||||
|
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||||
|
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||||
|
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||||
|
//
|
||||||
|
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||||
|
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||||
|
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||||
|
// having failed.
|
||||||
|
|
||||||
|
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||||
|
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||||
|
// upstream should branch on it.
|
||||||
|
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||||
|
references, err := inv.ToCollect(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(references) == 0 && len(kept) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||||
|
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||||
|
// nothing to collect.
|
||||||
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil || address == "" {
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||||
|
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||||
|
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||||
|
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||||
|
// tonight.
|
||||||
|
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||||
|
defer stop()
|
||||||
|
store := artifacts.Store{Address: address}
|
||||||
|
|
||||||
|
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||||
|
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||||
|
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||||
|
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||||
|
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||||
|
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||||
|
wrote, missing, err := holdKept(within, store, kept)
|
||||||
|
if wrote > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||||
|
}
|
||||||
|
if missing > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||||
|
"`collection` lists them\n", missing)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var done []string
|
||||||
|
var left, skipped int
|
||||||
|
for i, reference := range references {
|
||||||
|
if i >= mostPerSweep || within.Err() != nil {
|
||||||
|
left = len(references) - i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
err := store.LetGo(within, reference)
|
||||||
|
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||||
|
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||||
|
// again for ever.
|
||||||
|
done = append(done, reference)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||||
|
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||||
|
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||||
|
// reason to stop, because the store was never asked. One of these at the front of
|
||||||
|
// the oldest-first order ended every sweep until this.
|
||||||
|
skipped++
|
||||||
|
if skipped == 1 {
|
||||||
|
fmt.Fprintf(os.Stderr,
|
||||||
|
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||||
|
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||||
|
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||||
|
// front of whoever was building something.
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||||
|
reference, err)
|
||||||
|
left = len(references) - i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(done) > 0 {
|
||||||
|
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||||
|
// the sweep ran out of budget would mean asking about them again for ever.
|
||||||
|
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||||
|
len(done), err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||||
|
len(done))
|
||||||
|
}
|
||||||
|
if left > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||||
|
}
|
||||||
|
if skipped > 0 {
|
||||||
|
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||||
|
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||||
|
//
|
||||||
|
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||||
|
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||||
|
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||||
|
// (novox/hq issue 226).
|
||||||
|
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||||
|
for _, reference := range kept {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||||
|
}
|
||||||
|
did, err := store.Hold(ctx, reference)
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
if did {
|
||||||
|
wrote++
|
||||||
|
}
|
||||||
|
case errors.Is(err, artifacts.Gone):
|
||||||
|
missing++
|
||||||
|
case errors.Is(err, artifacts.ErrNotOurs):
|
||||||
|
default:
|
||||||
|
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return wrote, missing, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||||
|
// several times a day converges within days of this landing; small enough that no single build
|
||||||
|
// waits on the whole backlog.
|
||||||
|
const mostPerSweep = 200
|
||||||
|
|
||||||
|
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||||
|
const sweepBudget = 60 * time.Second
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/artifacts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||||
|
// (novox/hq issue 253, ADR 0189).
|
||||||
|
//
|
||||||
|
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||||
|
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||||
|
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||||
|
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||||
|
// dry run go.
|
||||||
|
//
|
||||||
|
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||||
|
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||||
|
// new verb in the seat's row.
|
||||||
|
|
||||||
|
type collectionReport struct {
|
||||||
|
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||||
|
Store string `json:"store"`
|
||||||
|
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||||
|
KeptArchives int `json:"kept_archives"`
|
||||||
|
// Held is how many of them the store holds by their manifest.
|
||||||
|
Held int `json:"held"`
|
||||||
|
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||||
|
Unheld []string `json:"unheld"`
|
||||||
|
// Missing are kept archives the store does not have at all.
|
||||||
|
Missing []string `json:"missing"`
|
||||||
|
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||||
|
Unasked int `json:"unasked"`
|
||||||
|
Stopped string `json:"stopped,omitempty"`
|
||||||
|
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||||
|
// collected — split by kind.
|
||||||
|
Eligible int `json:"eligible"`
|
||||||
|
EligibleImages int `json:"eligible_images"`
|
||||||
|
EligibleArchives int `json:"eligible_archives"`
|
||||||
|
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||||
|
SafeToCollect bool `json:"safe_to_collect"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func collectionCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "answer as JSON")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 0 {
|
||||||
|
return errors.New("collection [--json]")
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
kept, err := inv.KeptArchives(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
eligible, err := inv.ToCollect(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||||
|
for _, reference := range eligible {
|
||||||
|
if strings.Contains(reference, "/blobs/") {
|
||||||
|
report.EligibleArchives++
|
||||||
|
} else {
|
||||||
|
report.EligibleImages++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if report.Store == "" {
|
||||||
|
report.Unasked = len(kept)
|
||||||
|
report.Stopped = "this mesh has no artifact store on its network"
|
||||||
|
} else {
|
||||||
|
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||||
|
}
|
||||||
|
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||||
|
|
||||||
|
if *asJSON {
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", " ")
|
||||||
|
return encoder.Encode(report)
|
||||||
|
}
|
||||||
|
printCollection(report)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||||
|
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||||
|
// identical failures says less than one line.
|
||||||
|
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||||
|
for i, reference := range kept {
|
||||||
|
held, err := store.Held(ctx, reference)
|
||||||
|
switch {
|
||||||
|
case err == nil && held:
|
||||||
|
report.Held++
|
||||||
|
case err == nil:
|
||||||
|
report.Unheld = append(report.Unheld, reference)
|
||||||
|
case errors.Is(err, artifacts.Gone):
|
||||||
|
report.Missing = append(report.Missing, reference)
|
||||||
|
case errors.Is(err, artifacts.ErrNotOurs):
|
||||||
|
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||||
|
report.Unasked++
|
||||||
|
default:
|
||||||
|
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return report.Unasked, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func printCollection(r collectionReport) {
|
||||||
|
store := r.Store
|
||||||
|
if store == "" {
|
||||||
|
store = "(not on the network)"
|
||||||
|
}
|
||||||
|
fmt.Printf("artifact store %s\n", store)
|
||||||
|
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||||
|
fmt.Printf(" held %d\n", r.Held)
|
||||||
|
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||||
|
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||||
|
if r.Unasked > 0 {
|
||||||
|
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||||
|
}
|
||||||
|
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||||
|
if len(r.Unheld) > 0 {
|
||||||
|
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||||
|
for _, reference := range r.Unheld {
|
||||||
|
fmt.Printf(" %s\n", reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(r.Missing) > 0 {
|
||||||
|
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||||
|
for _, reference := range r.Missing {
|
||||||
|
fmt.Printf(" %s\n", reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
if r.SafeToCollect {
|
||||||
|
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||||
|
} else {
|
||||||
|
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||||
|
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||||
|
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||||
|
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||||
|
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||||
|
|
||||||
|
func aTalker() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||||
|
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, aTalker())
|
||||||
|
|
||||||
|
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||||
|
// stands and says exactly what must happen before a push — never silently.
|
||||||
|
said, err := assign(ctx, open, "laptop", "talker")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||||
|
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||||
|
plan, settings, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||||
|
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||||
|
// does not mint again: a credential rotates on purpose, never by habit.
|
||||||
|
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err = assign(ctx, open, "laptop", "talker")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(said, "module issue") {
|
||||||
|
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||||
|
}
|
||||||
|
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if again != hash {
|
||||||
|
t.Fatal("re-assigning rotated the credential")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||||
|
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
register(t, open, helloWeb())
|
||||||
|
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(said, "credential") {
|
||||||
|
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||||
|
type recordingDelivery struct {
|
||||||
|
did []string
|
||||||
|
grantErr error
|
||||||
|
declareErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||||
|
for _, s := range sending {
|
||||||
|
r.did = append(r.did, "grant "+s.node)
|
||||||
|
}
|
||||||
|
return r.grantErr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||||
|
if r.declareErr != nil {
|
||||||
|
return "", r.declareErr
|
||||||
|
}
|
||||||
|
r.did = append(r.did, "declare "+s.node)
|
||||||
|
return "digest-" + s.node, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ready(names ...string) []readyNode {
|
||||||
|
var out []readyNode
|
||||||
|
for _, n := range names {
|
||||||
|
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||||
|
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||||
|
// carries the controller's own right to issue them, and goes first.
|
||||||
|
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||||
|
d := &recordingDelivery{}
|
||||||
|
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||||
|
if !reflect.DeepEqual(d.did, want) {
|
||||||
|
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||||
|
}
|
||||||
|
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||||
|
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||||
|
}
|
||||||
|
|
||||||
|
held := &recordingDelivery{}
|
||||||
|
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||||
|
if !reflect.DeepEqual(held.did, want) {
|
||||||
|
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||||
|
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||||
|
// grant that would let the controller issue memberships is never held behind them.
|
||||||
|
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||||
|
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||||
|
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||||
|
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||||
|
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||||
|
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||||
|
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||||
|
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A membership that failed for one machine: that machine alone.
|
||||||
|
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||||
|
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||||
|
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||||
|
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||||
|
}
|
||||||
|
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||||
|
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The announced upgrade that hit it is asked again.
|
||||||
|
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||||
|
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||||
|
}
|
||||||
|
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||||
|
t.Fatal("any failure is asked again, not only a grant's")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing to send is nothing granted either.
|
||||||
|
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||||
|
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||||
|
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||||
|
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||||
|
list := "users: [a, b]"
|
||||||
|
if userListBehind(list, digestOf([]byte(list))) {
|
||||||
|
t.Fatal("the list it was sent reads as behind")
|
||||||
|
}
|
||||||
|
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||||
|
t.Fatal("a changed or never-sent list reads as current")
|
||||||
|
}
|
||||||
|
if userListBehind("", "") {
|
||||||
|
t.Fatal("a machine sent no list reads as behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||||
|
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||||
|
// when it is behind.
|
||||||
|
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
names []string
|
||||||
|
holder string
|
||||||
|
behind bool
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||||
|
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||||
|
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||||
|
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||||
|
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||||
|
} {
|
||||||
|
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
|
||||||
|
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
|
||||||
|
// module built on one of those moves as well.
|
||||||
|
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
|
||||||
|
entry := func(name string) inventory.Entry {
|
||||||
|
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
|
||||||
|
}
|
||||||
|
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
|
||||||
|
against := map[string][]string{
|
||||||
|
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||||
|
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
|
||||||
|
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||||
|
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
|
||||||
|
}
|
||||||
|
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
|
||||||
|
var names []string
|
||||||
|
for _, e := range got {
|
||||||
|
names = append(names, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
|
||||||
|
if len(names) != len(want) {
|
||||||
|
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
|
||||||
|
}
|
||||||
|
for _, n := range names {
|
||||||
|
if !want[n] {
|
||||||
|
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The dependents come in base order when the merge orders them: the runtime, then shop, then
|
||||||
|
// the plugin that stands on shop.
|
||||||
|
ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against)
|
||||||
|
pos := map[string]int{}
|
||||||
|
for i, e := range ordered {
|
||||||
|
pos[e.Manifest.Module] = i
|
||||||
|
}
|
||||||
|
if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) {
|
||||||
|
t.Fatalf("not in base order: %v", ordered)
|
||||||
|
}
|
||||||
|
// Nothing moved: nothing follows.
|
||||||
|
if more := dependentsOf(nil, entries, against); len(more) != 0 {
|
||||||
|
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||||
|
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||||
|
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||||
|
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||||
|
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||||
|
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||||
|
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||||
|
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||||
|
var asked link.BuildRequest
|
||||||
|
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||||
|
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||||
|
var answered link.BuildResult
|
||||||
|
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||||
|
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||||
|
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||||
|
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||||
|
// through sendTo does not wait on itself.
|
||||||
|
type heldKey struct{}
|
||||||
|
|
||||||
|
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||||
|
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||||
|
// Release gives back only what this call took.
|
||||||
|
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||||
|
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||||
|
var take []string
|
||||||
|
for _, n := range names {
|
||||||
|
if !already[n] {
|
||||||
|
take = append(take, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(take) == 0 {
|
||||||
|
return ctx, func() {}, nil
|
||||||
|
}
|
||||||
|
release, err := open.inventory.HoldNodes(ctx, take)
|
||||||
|
if err != nil {
|
||||||
|
return ctx, nil, err
|
||||||
|
}
|
||||||
|
held := map[string]bool{}
|
||||||
|
for n := range already {
|
||||||
|
held[n] = true
|
||||||
|
}
|
||||||
|
for _, n := range take {
|
||||||
|
held[n] = true
|
||||||
|
}
|
||||||
|
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled
|
||||||
|
// and a send that fails leave nobody waiting for the node.
|
||||||
|
func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
unmarshallable := func(context.Context, string) (sendable, error) {
|
||||||
|
return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil
|
||||||
|
}
|
||||||
|
plain := func(context.Context, string) (sendable, error) {
|
||||||
|
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||||
|
}
|
||||||
|
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||||
|
fine := &recordingDelivery{}
|
||||||
|
|
||||||
|
for name, round := range map[string]func() error{
|
||||||
|
"a body that cannot be marshalled": func() error {
|
||||||
|
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
"a send that fails": func() error {
|
||||||
|
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
if err := round(); err == nil {
|
||||||
|
t.Fatalf("%s was not an error", name)
|
||||||
|
}
|
||||||
|
waiting, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
release, err := open.inventory.HoldNodes(waiting, []string{"anchor"})
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("after %s the node is still held: %v", name, err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||||
|
// as holding.
|
||||||
|
//
|
||||||
|
// **A seat held by derivation is a seat held by accident of being alone** (novox/hq
|
||||||
|
// 04-ISSUES/170). ADR 0131 lets a holder on record settle a seat, and lets any other assignment
|
||||||
|
// whose module could hold it stand beside the holder, eligible and silent. But a seat nobody
|
||||||
|
// ever handed over has no record, so its holder is whichever assignment happened to be the sole
|
||||||
|
// claimant — and the day a second one is assigned, both claim, both are refused, and the first
|
||||||
|
// one's whole machine stops resolving. That is what assigning a second postgres did to the
|
||||||
|
// control plane's own store.
|
||||||
|
//
|
||||||
|
// So the mesh writes the derived answer down before it acts on an assignment: for every
|
||||||
|
// mesh-scoped seat with exactly one resolved holder and nothing on record, that holder is
|
||||||
|
// recorded as the standing one — the same record `seat <name> --to <node>/<module>` makes by
|
||||||
|
// hand, made from what the mesh already resolved. A seat with two derived claimants is left
|
||||||
|
// alone: that is the ambiguity a person settles, and recording either would be guessing.
|
||||||
|
//
|
||||||
|
// Node-scoped seats are untouched: a record is one holder per seat, and a node-scoped seat has
|
||||||
|
// one holder per machine (ADR 0121), so there is nothing for a record to settle there.
|
||||||
|
func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// exclude nobody: every node's claims, resolved with the holdings on record.
|
||||||
|
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recorded, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// A record is a row against a seat the store knows. A seat it does not — a mesh whose seats
|
||||||
|
// were never seeded, a seat a module declares for itself — stays held by derivation, as it
|
||||||
|
// always was; a missing row is not a reason an assignment fails.
|
||||||
|
known, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
recordable := map[string]bool{}
|
||||||
|
for _, s := range known {
|
||||||
|
recordable[s.Name] = true
|
||||||
|
}
|
||||||
|
onRecord := map[string]bool{}
|
||||||
|
for _, h := range recorded {
|
||||||
|
if s, ok := catalogue.SeatNamed(h.Claim); ok {
|
||||||
|
onRecord[s.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
holders := map[string][]catalogue.Held{}
|
||||||
|
for _, h := range world.Held {
|
||||||
|
if h.Scope != catalogue.ScopeMesh {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s, ok := catalogue.SeatNamed(h.Claim)
|
||||||
|
if !ok || onRecord[s.Name] || !recordable[s.Name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
holders[s.Name] = append(holders[s.Name], h)
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(holders))
|
||||||
|
for name := range holders {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
var said []string
|
||||||
|
for _, name := range names {
|
||||||
|
if len(holders[name]) != 1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
h := holders[name][0]
|
||||||
|
if err := inv.HoldSeat(ctx, name, catalogue.ScopeMesh, h.Node, h.Module); err != nil {
|
||||||
|
return said, err
|
||||||
|
}
|
||||||
|
said = append(said, fmt.Sprintf(
|
||||||
|
"recorded %s on %s as the standing holder of %s, which it held only by being alone",
|
||||||
|
h.Module, h.Node, name))
|
||||||
|
}
|
||||||
|
return said, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||||
|
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||||
|
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||||
|
//
|
||||||
|
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||||
|
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||||
|
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||||
|
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||||
|
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||||
|
// answers is a lookup that waits out its timeout on every name.
|
||||||
|
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||||
|
var replicated []catalogue.Seat
|
||||||
|
for _, s := range catalogue.Seats() {
|
||||||
|
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||||
|
replicated = append(replicated, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(replicated) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
recorded, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address := map[string]string{}
|
||||||
|
for _, p := range places {
|
||||||
|
address[p.Name] = p.Address
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]map[string]string{}
|
||||||
|
for _, seat := range replicated {
|
||||||
|
var nodes []string
|
||||||
|
for _, h := range recorded {
|
||||||
|
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||||
|
nodes = append(nodes, h.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
var derived []string
|
||||||
|
for _, e := range entries {
|
||||||
|
for _, c := range e.Manifest.Claims {
|
||||||
|
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||||
|
derived = append(derived, e.On...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(derived) == 1 {
|
||||||
|
nodes = derived
|
||||||
|
}
|
||||||
|
}
|
||||||
|
at := map[string]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if address[n] != "" {
|
||||||
|
at[overlay.InternalName(n)] = address[n]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out[seat.Name] = at
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A seat nobody ever handed over is held by whichever assignment happened to be alone — and the
|
||||||
|
// day a second module able to hold it is assigned, both claimed, both were refused, and the first
|
||||||
|
// one's machine stopped resolving (novox/hq 04-ISSUES/170). The mesh now writes the derived holder
|
||||||
|
// down before it acts, so the second assignment stands beside the holder on record.
|
||||||
|
|
||||||
|
func aSeatedStore() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASecondEligibleHolderStandsBesideTheOneHeldByBeingAlone(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
// Every deploy seeds the mesh's own seats; a record is a row against one of them.
|
||||||
|
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, aSeatedStore())
|
||||||
|
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := assign(ctx, open, "laptop", "store")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a second store, eligible for the seat, was refused:\n%s\n%v", said, err)
|
||||||
|
}
|
||||||
|
if strings.Contains(said, "cannot be worked out") {
|
||||||
|
t.Fatalf("assigning a second store unsettled the first one's machine:\n%s", said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "recorded store on anchor as the standing holder of mesh-store") {
|
||||||
|
t.Fatalf("the holder by derivation was not written down:\n%s", said)
|
||||||
|
}
|
||||||
|
|
||||||
|
holdings, err := open.inventory.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, h := range holdings {
|
||||||
|
if h.Claim == "mesh-store" {
|
||||||
|
found = true
|
||||||
|
if h.Node != "anchor" || h.Module != "store" {
|
||||||
|
t.Fatalf("mesh-store is recorded on %s/%s, not on the one that held it", h.Node, h.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("mesh-store has no holder on record after assigning: %v", holdings)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the record decides from here: the anchor's plan holds the seat, the laptop's does not.
|
||||||
|
for node, holds := range map[string]bool{"anchor": true, "laptop": false} {
|
||||||
|
plan, _, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s no longer resolves: %v", node, err)
|
||||||
|
}
|
||||||
|
var claimed bool
|
||||||
|
for _, c := range plan.Claims {
|
||||||
|
if c.Claim == "mesh-store" {
|
||||||
|
claimed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if claimed != holds {
|
||||||
|
t.Fatalf("%s holds mesh-store: %v, want %v", node, claimed, holds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHolderOnRecordIsNotRewrittenByDerivation(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
register(t, open, aSeatedStore())
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := assign(ctx, open, node, "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A person hands the seat to the laptop. From here the record decides, and what the mesh
|
||||||
|
// derives must never write over it.
|
||||||
|
if err := open.inventory.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "laptop", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := recordDerivedHolders(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(said) != 0 {
|
||||||
|
t.Fatalf("a seat on record was written again from derivation: %v", said)
|
||||||
|
}
|
||||||
|
holdings, _ := open.inventory.Holdings(ctx)
|
||||||
|
for _, h := range holdings {
|
||||||
|
if h.Claim == "mesh-store" && h.Node != "laptop" {
|
||||||
|
t.Fatalf("the record moved to %s", h.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A host refuses a declaration carrying a field it does not know, and refuses it whole — so every new
|
||||||
|
// field is a flag day, and the mesh had no record of which host any machine ran (novox/hq
|
||||||
|
// 04-ISSUES/087). The order was kept by somebody remembering it.
|
||||||
|
|
||||||
|
func TestTheMeshNamesWhichMachinesRunWhichHost(t *testing.T) {
|
||||||
|
split := hostSplit([]inventory.Node{
|
||||||
|
{Name: "anchor", HostVersion: "04a27ca"},
|
||||||
|
{Name: "laptop", HostVersion: "ced54d4"},
|
||||||
|
{Name: "spare", HostVersion: "04a27ca"},
|
||||||
|
})
|
||||||
|
if len(split) != 2 {
|
||||||
|
t.Fatalf("two versions were reported and the split has %d: %v", len(split), split)
|
||||||
|
}
|
||||||
|
if got := strings.Join(split["04a27ca"], ","); got != "anchor,spare" && got != "spare,anchor" {
|
||||||
|
t.Fatalf("04a27ca is held by %q", got)
|
||||||
|
}
|
||||||
|
if got := strings.Join(split["ced54d4"], ","); got != "laptop" {
|
||||||
|
t.Fatalf("ced54d4 is held by %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheMeshDoesNotClaimWhichHostIsNewer(t *testing.T) {
|
||||||
|
// **The fault this replaced.** A host reports its version as a commit, and commits have no order.
|
||||||
|
// The first version compared them as strings and, on the live mesh, named the three machines
|
||||||
|
// running the NEWER host as the ones behind: `ced54d4` sorts above `04a27ca` and means nothing.
|
||||||
|
//
|
||||||
|
// There is no assertion to make about which is newer, and that is the point — the type says so.
|
||||||
|
// hostSplit returns who runs what, and nothing that could be read as an ordering.
|
||||||
|
split := hostSplit([]inventory.Node{
|
||||||
|
{Name: "old-but-sorts-high", HostVersion: "ced54d4"},
|
||||||
|
{Name: "new-but-sorts-low", HostVersion: "04a27ca"},
|
||||||
|
})
|
||||||
|
for version, machines := range split {
|
||||||
|
if len(machines) != 1 {
|
||||||
|
t.Fatalf("%s is held by %v", version, machines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMachineThatHasNotSaidIsNotAVersion(t *testing.T) {
|
||||||
|
// It may be running anything. Counting it as a version would invent a disagreement; `node show`
|
||||||
|
// says per machine that it has not said.
|
||||||
|
split := hostSplit([]inventory.Node{
|
||||||
|
{Name: "anchor", HostVersion: "04a27ca"},
|
||||||
|
{Name: "quiet"},
|
||||||
|
})
|
||||||
|
if split != nil {
|
||||||
|
t.Fatalf("one reported version and one silence read as a disagreement: %v", split)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMachinesAgreeingOnTheirHostAreNotADisagreement(t *testing.T) {
|
||||||
|
if split := hostSplit([]inventory.Node{
|
||||||
|
{Name: "anchor", HostVersion: "v2"},
|
||||||
|
{Name: "laptop", HostVersion: "v2"},
|
||||||
|
}); split != nil {
|
||||||
|
t.Fatalf("machines agreeing reported a split: %v", split)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
|
||||||
|
if split := hostSplit([]inventory.Node{{Name: "anchor"}, {Name: "laptop"}}); split != nil {
|
||||||
|
t.Fatalf("a mesh told no host version reported a split: %v", split)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
|
||||||
|
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
|
||||||
|
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
|
||||||
|
// fault was a field that existed on one side of the wire only.
|
||||||
|
open := aMesh(t)
|
||||||
|
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if record.HostVersion != "" {
|
||||||
|
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
again, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if again.HostVersion != "ced54d4" {
|
||||||
|
t.Fatalf("the reported host version read back as %q", again.HostVersion)
|
||||||
|
}
|
||||||
|
// An empty report never clears what a machine last said: a bare word that the node is there says
|
||||||
|
// nothing about its host.
|
||||||
|
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if kept.HostVersion != "ced54d4" {
|
||||||
|
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
|
||||||
|
kept.HostVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||||
|
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||||
|
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||||
|
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||||
|
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||||
|
// compositions, and the host's refusal does what it is for.
|
||||||
|
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||||
|
allot := numbered()
|
||||||
|
var composed []string
|
||||||
|
compose := func(stamp string) func(string) (sendable, error) {
|
||||||
|
return func(node string) (sendable, error) {
|
||||||
|
composed = append(composed, stamp)
|
||||||
|
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Composed first — before an assignment changed — and sent last.
|
||||||
|
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||||
|
// Composed after the change, sent first.
|
||||||
|
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||||
|
|
||||||
|
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||||
|
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||||
|
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||||
|
}
|
||||||
|
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||||
|
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||||
|
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||||
|
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||||
|
}
|
||||||
|
if len(composed) != 2 || composed[0] != "before" {
|
||||||
|
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||||
|
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||||
|
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
allot := func(node string) (int64, error) {
|
||||||
|
if node == "anchor" {
|
||||||
|
return 0, context.DeadlineExceeded
|
||||||
|
}
|
||||||
|
return 7, nil
|
||||||
|
}
|
||||||
|
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||||
|
calls++
|
||||||
|
if node == "anchor" {
|
||||||
|
t.Fatal("anchor was composed although its number could not be taken")
|
||||||
|
}
|
||||||
|
return sendable{}, nil
|
||||||
|
})
|
||||||
|
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||||
|
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||||
|
}
|
||||||
|
if len(refusals) != 1 {
|
||||||
|
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||||
|
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||||
|
// current" over a machine that had just been sent something else.
|
||||||
|
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||||
|
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||||
|
digest, err := recordSent(ctx, inv, "anchor", body)
|
||||||
|
if err != nil {
|
||||||
|
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||||
|
// through the detached context, so look the node up again on a live one.
|
||||||
|
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||||
|
}
|
||||||
|
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if outstanding != digest || digest != digestOf(body) {
|
||||||
|
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,481 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// licenceCommand is everything about model access the mesh holds.
|
||||||
|
//
|
||||||
|
// **A licence is a named thing and the name is the operator's** (novox/hq ADR 0024). *The personal
|
||||||
|
// account*, *the organisation's account* — those are names a person uses, and the mesh has to use
|
||||||
|
// them too, because the whole point is saying which one a given consumer uses.
|
||||||
|
func licenceCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New(
|
||||||
|
"licence add|list|use|release|key|manager|set-grant|refresh|submit-refresh|forget")
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "add":
|
||||||
|
return licenceAdd(ctx, args[1:])
|
||||||
|
case "list":
|
||||||
|
return licenceList(ctx)
|
||||||
|
case "use":
|
||||||
|
return licenceUse(ctx, args[1:], true)
|
||||||
|
case "release":
|
||||||
|
return licenceUse(ctx, args[1:], false)
|
||||||
|
case "key":
|
||||||
|
return licenceKey(ctx, args[1:])
|
||||||
|
case "manager":
|
||||||
|
return licenceManager(ctx, args[1:])
|
||||||
|
case "set-grant":
|
||||||
|
return licenceSetGrant(ctx, args[1:])
|
||||||
|
case "refresh":
|
||||||
|
return licenceRefresh(ctx, args[1:])
|
||||||
|
case "submit-refresh":
|
||||||
|
return licenceSubmitRefresh(ctx, args[1:])
|
||||||
|
case "forget":
|
||||||
|
return licenceForget(ctx, args[1:])
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"licence %q; it is add, list, use, release, key, manager, set-grant, refresh, "+
|
||||||
|
"submit-refresh or forget", args[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
func licenceAdd(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("licence add", flag.ContinueOnError)
|
||||||
|
// What a consumer must know that is not secret — a base URL, a model name. Never the key.
|
||||||
|
serves := set.String("serves", "",
|
||||||
|
"JSON a consumer must know that is not secret, such as a base URL or a model")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 2 {
|
||||||
|
return errors.New(`licence add <vendor> <name> [--serves '{"model":"..."}']`)
|
||||||
|
}
|
||||||
|
vendor, name := positionals[0], positionals[1]
|
||||||
|
|
||||||
|
values := map[string]any{}
|
||||||
|
if strings.TrimSpace(*serves) != "" {
|
||||||
|
if err := json.Unmarshal([]byte(*serves), &values); err != nil {
|
||||||
|
return fmt.Errorf("--serves is not JSON: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
if err := held.Add(ctx, name, vendor, values); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s (%s) recorded. Nothing uses it yet, and it has no key:\n"+
|
||||||
|
" licence use %s <node> <module>\n licence key %s\n", name, vendor, name, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func licenceList(ctx context.Context) error {
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
|
||||||
|
all, err := held.All(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(all) == 0 {
|
||||||
|
// Said, not printed as nothing: an empty list and a failed read must never look the same.
|
||||||
|
fmt.Println("this mesh holds no licences")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, one := range all {
|
||||||
|
holders, err := held.HoldersOf(ctx, one.Name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s (%s)\n", one.Name, one.Vendor)
|
||||||
|
if len(holders) == 0 {
|
||||||
|
fmt.Printf(" nobody uses it\n")
|
||||||
|
}
|
||||||
|
for _, h := range holders {
|
||||||
|
// Whether it has a key is the question somebody is actually asking, so it is said
|
||||||
|
// per holder rather than per licence: the key was sealed to the holders that existed
|
||||||
|
// when it was supplied, and one recorded afterwards has none.
|
||||||
|
state := "has no key — supply it again with `licence key " + one.Name + "`"
|
||||||
|
if h.Sealed != "" {
|
||||||
|
state = "has a key"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s on %s: %s\n", h.Module, h.Node, state)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func licenceUse(ctx context.Context, args []string, using bool) error {
|
||||||
|
verb := "use"
|
||||||
|
if !using {
|
||||||
|
verb = "release"
|
||||||
|
}
|
||||||
|
if len(args) != 3 {
|
||||||
|
return fmt.Errorf("licence %s <name> <node> <module>", verb)
|
||||||
|
}
|
||||||
|
name, node, module := args[0], args[1], args[2]
|
||||||
|
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
|
||||||
|
if !using {
|
||||||
|
if err := held.StopUsing(ctx, name, node, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s no longer uses %s. Its copy of the key goes on the next push\n",
|
||||||
|
module, node, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := held.Use(ctx, name, node, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s uses %s.\n", module, node, name)
|
||||||
|
// The consequence, said now rather than discovered as a machine that resolves and receives
|
||||||
|
// nothing: the mesh discarded the plaintext, so a holder added after the key was supplied has
|
||||||
|
// no key and the mesh cannot make one.
|
||||||
|
sealed, err := held.KeyFor(ctx, name, node, module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if sealed == "" {
|
||||||
|
fmt.Printf(" It has no key yet — the mesh discarded the plaintext when it was supplied "+
|
||||||
|
"and cannot seal another. Supply it again:\n licence key %s\n", name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// licenceKey is the *accept* verb novox/hq ADR 0024 names as missing.
|
||||||
|
//
|
||||||
|
// Take a value, seal it to each holder, and discard the plaintext. Every other credential the
|
||||||
|
// mesh handles it generated itself; an API key arrives from a person, and a mesh that kept
|
||||||
|
// operator-supplied keys readably is the arrangement this project measured and rejected.
|
||||||
|
func licenceKey(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("licence key", flag.ContinueOnError)
|
||||||
|
// A file rather than an argument, by default. A key on a command line is a key in shell
|
||||||
|
// history and in every process listing taken while it ran.
|
||||||
|
from := set.String("file", "", "read the key from a file instead of standard input")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("licence key <name> [--file <path>]")
|
||||||
|
}
|
||||||
|
name := positionals[0]
|
||||||
|
|
||||||
|
var value string
|
||||||
|
if *from != "" {
|
||||||
|
raw, err := os.ReadFile(*from)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
value = strings.TrimSpace(string(raw))
|
||||||
|
} else {
|
||||||
|
fmt.Fprintln(os.Stderr, "reading the key from standard input; it is not echoed anywhere")
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
line, err := reader.ReadString('\n')
|
||||||
|
if err != nil && line == "" {
|
||||||
|
return fmt.Errorf("nothing was given on standard input: %w", err)
|
||||||
|
}
|
||||||
|
value = strings.TrimSpace(line)
|
||||||
|
}
|
||||||
|
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
sealed, err := held.Accept(ctx, name, value, func(node string) (string, error) {
|
||||||
|
return inv.SealingKeyOf(ctx, node)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
if sealed > 0 {
|
||||||
|
// Some holders got it and some did not, and the person holding the key is the only
|
||||||
|
// one who can finish the job. Saying how far it got is the difference between running
|
||||||
|
// this again knowing what it will do and running it hoping.
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w\n\n%d holder(s) were sealed before this. Running `licence key %s` again "+
|
||||||
|
"with the same key seals the rest and changes nothing for those already done",
|
||||||
|
err, sealed, name)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Not echoed back, ever. What is stored is unreadable by whoever holds it, the mesh included,
|
||||||
|
// and printing the value here would put the one copy that matters on a terminal.
|
||||||
|
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
|
||||||
|
sealed)
|
||||||
|
fmt.Printf(" run `push` to deliver it\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// licenceManager names the one node that holds a refreshable-grant licence's refresh token readably
|
||||||
|
// and refreshes it centrally (novox/hq ADR 0050).
|
||||||
|
//
|
||||||
|
// **Only a refreshable-grant licence has one.** A static-key licence has no refresh token to hold, so
|
||||||
|
// naming a manager for it is refused where the mistake is made rather than kept as a field that means
|
||||||
|
// nothing — the absent manager is part of what keeps a static key from ever holding a value readably
|
||||||
|
// at rest.
|
||||||
|
func licenceManager(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 3 {
|
||||||
|
return errors.New("licence manager <name> <node> <module>")
|
||||||
|
}
|
||||||
|
name, node, module := args[0], args[1], args[2]
|
||||||
|
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
if err := held.SetManager(ctx, name, node, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s holds and refreshes %s.\n"+
|
||||||
|
" Its refresh token is sealed to %s's key — readable by that node alone, not by any other "+
|
||||||
|
"node and not by this database. Put %s on the licence too so it is delivered the token:\n"+
|
||||||
|
" licence use %s %s %s\n", module, node, name, node, module, name, node, module)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sealedGrantJSON is the wire shape of a sealed refresh token on this command surface: an anonymous
|
||||||
|
// sealed box and the public key it was sealed to, and nothing else.
|
||||||
|
//
|
||||||
|
// **Every field of it is ciphertext or a public key.** `sealed` is the refresh token as a
|
||||||
|
// `crypto_box_seal` to the manager node's public key; `manager_key` is that public key. Neither is
|
||||||
|
// the refresh token in the clear — which is why this surface may read one in (`set-grant`,
|
||||||
|
// `submit-refresh`) without the control plane ever holding a refresh token it could read. The manager
|
||||||
|
// module, on the manager node, seals it; the HOST, on that node, unseals it to deliver cleartext. This
|
||||||
|
// database, and this surface, only ever forward the box (novox/hq ADR 0050).
|
||||||
|
type sealedGrantJSON struct {
|
||||||
|
Sealed string `json:"sealed"`
|
||||||
|
ManagerKey string `json:"manager_key"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// readSealedGrant reads a sealed refresh token from a file or standard input as JSON.
|
||||||
|
func readSealedGrant(from string) (sealedGrantJSON, error) {
|
||||||
|
var raw []byte
|
||||||
|
var err error
|
||||||
|
if from != "" {
|
||||||
|
raw, err = os.ReadFile(from)
|
||||||
|
} else {
|
||||||
|
raw, err = readAllStdin()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return sealedGrantJSON{}, err
|
||||||
|
}
|
||||||
|
var g sealedGrantJSON
|
||||||
|
if err := json.Unmarshal(raw, &g); err != nil {
|
||||||
|
return sealedGrantJSON{}, fmt.Errorf("the sealed refresh token is not JSON: %w", err)
|
||||||
|
}
|
||||||
|
if g.Sealed == "" || g.ManagerKey == "" {
|
||||||
|
return sealedGrantJSON{}, errors.New(
|
||||||
|
"a sealed refresh token is {sealed, manager_key}, and one part is missing")
|
||||||
|
}
|
||||||
|
return g, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readAllStdin() ([]byte, error) {
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
return io.ReadAll(reader)
|
||||||
|
}
|
||||||
|
|
||||||
|
// licenceSetGrant stores a sealed refresh token the manager module produced — adoption, and the
|
||||||
|
// re-seal after a rotation done outside this process (novox/hq ADR 0050).
|
||||||
|
//
|
||||||
|
// **It takes a sealed box, never a refresh token.** The manager module, on the manager node, reads
|
||||||
|
// the operator's refresh token, seals it to that node's own public key, and hands the box here. So the
|
||||||
|
// one moment a refresh token is in the clear is on the manager node, never in the control plane — the
|
||||||
|
// same bound the whole carve-out keeps. This surface refuses anything that is not a complete sealed
|
||||||
|
// grant rather than storing half of one.
|
||||||
|
func licenceSetGrant(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("licence set-grant", flag.ContinueOnError)
|
||||||
|
from := set.String("file", "", "read the sealed refresh token from a file instead of standard input")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("licence set-grant <name> [--file <path>]")
|
||||||
|
}
|
||||||
|
name := positionals[0]
|
||||||
|
|
||||||
|
grant, err := readSealedGrant(*from)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
if err := held.SetRefreshGrant(ctx, name, grant.Sealed, grant.ManagerKey); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
|
||||||
|
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n",
|
||||||
|
"the manager", name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// licenceSubmitRefresh publishes a refresh a MANAGER NODE already performed: the new access token is
|
||||||
|
// sealed to every holder, and a rotated refresh token replaces the stored envelope (novox/hq ADR
|
||||||
|
// 0050, Phase C).
|
||||||
|
//
|
||||||
|
// **This is the boundary the invariant rests on.** The manager runtime, on the manager node, opened
|
||||||
|
// the at-rest envelope with that node's key, called the vendor's OAuth endpoint, and produced this:
|
||||||
|
// the new access token in the clear, and — only if the vendor rotated it — the refresh token already
|
||||||
|
// re-sealed at rest. This reads exactly those two things and no refresh token in the clear ever
|
||||||
|
// reaches it, because it is never given one. The access token is sealed per holder and discarded,
|
||||||
|
// as any accepted key is; the rotated envelope is stored opaque.
|
||||||
|
func licenceSubmitRefresh(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("licence submit-refresh", flag.ContinueOnError)
|
||||||
|
accessFrom := set.String("access-file", "",
|
||||||
|
"read the new access token from a file instead of standard input")
|
||||||
|
grantFrom := set.String("grant-file", "",
|
||||||
|
"the rotated sealed refresh token, if the vendor rotated it; omit if it did not")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New(
|
||||||
|
"licence submit-refresh <name> [--access-file <path>] [--grant-file <path>]")
|
||||||
|
}
|
||||||
|
name := positionals[0]
|
||||||
|
|
||||||
|
var accessToken string
|
||||||
|
if *accessFrom != "" {
|
||||||
|
raw, err := os.ReadFile(*accessFrom)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
accessToken = strings.TrimSpace(string(raw))
|
||||||
|
} else {
|
||||||
|
raw, err := readAllStdin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
accessToken = strings.TrimSpace(string(raw))
|
||||||
|
}
|
||||||
|
if accessToken == "" {
|
||||||
|
return errors.New("no access token was given, so there is nothing to seal")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The rotated sealed token is optional: absent, the stored refresh token is left exactly as it was.
|
||||||
|
var newSealed, newManagerKey string
|
||||||
|
if *grantFrom != "" {
|
||||||
|
grant, err := readSealedGrant(*grantFrom)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
newSealed, newManagerKey = grant.Sealed, grant.ManagerKey
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
held, err := open.Licences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
sealed, err := held.SubmitRefresh(ctx, name, accessToken, newSealed, newManagerKey,
|
||||||
|
func(node string) (string, error) {
|
||||||
|
return inv.SealingKeyOf(ctx, node)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rotatedNote := "the refresh token was left with its manager unchanged"
|
||||||
|
if newSealed != "" {
|
||||||
|
rotatedNote = "the rotated refresh token replaced the stored box, still readable by the " +
|
||||||
|
"manager node alone"
|
||||||
|
}
|
||||||
|
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
|
||||||
|
" run `push` to deliver it\n", name, sealed, rotatedNote)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// licenceRefresh mints a new access token for a refreshable-grant licence and seals it to every
|
||||||
|
// holder (novox/hq ADR 0050). The refresh token stays with the manager and is never delivered.
|
||||||
|
//
|
||||||
|
// The vendor's actual refresh is a plug-in this build does not ship (Phase C), so here this reports
|
||||||
|
// that plainly rather than pretending to have refreshed.
|
||||||
|
func licenceRefresh(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("licence refresh <name>")
|
||||||
|
}
|
||||||
|
name := args[0]
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
held, err := open.Licences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
sealed, err := held.Refresh(ctx, name, func(node string) (string, error) {
|
||||||
|
return inv.SealingKeyOf(ctx, node)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
|
||||||
|
"with its manager.\n run `push` to deliver it\n", name, sealed)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func licenceForget(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("licence forget <name>")
|
||||||
|
}
|
||||||
|
held, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer held.Close()
|
||||||
|
if err := held.Forget(ctx, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Said plainly, because the mesh cannot do it and pretending otherwise is worse than useless:
|
||||||
|
// a licence outliving its holder is a live credential nobody is watching.
|
||||||
|
fmt.Printf("%s is forgotten, and every record of who held it with it.\n"+
|
||||||
|
" The key itself is not the mesh's to revoke — do that where the licence was bought\n",
|
||||||
|
args[0])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
// Command mesh-controller is the control plane: everything that needs to know about more than one
|
||||||
|
// node (novox/hq ADR 0006).
|
||||||
|
//
|
||||||
|
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
|
||||||
|
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
|
||||||
|
// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"github.com/novox/mesh-controller/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// version is stamped at link time. Unset in a development build, and it says so rather than
|
||||||
|
// claiming a number.
|
||||||
|
var version = "development build"
|
||||||
|
|
||||||
|
// held is a context this process was granted, and the schema it carries.
|
||||||
|
//
|
||||||
|
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
|
||||||
|
// yet, not because they are optional.
|
||||||
|
var held = []struct {
|
||||||
|
name string
|
||||||
|
migrations func() ([]store.Migration, error)
|
||||||
|
}{
|
||||||
|
{inventory.Name, inventory.Migrations},
|
||||||
|
{identity.Name, identity.Migrations},
|
||||||
|
{licences.Name, licences.Migrations},
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := run(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run() error {
|
||||||
|
args := os.Args[1:]
|
||||||
|
if len(args) == 0 {
|
||||||
|
usage()
|
||||||
|
return fmt.Errorf("no command given")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "build":
|
||||||
|
return buildCommand(ctx, args[1:])
|
||||||
|
case "builder":
|
||||||
|
return builderCommand(ctx, args[1:])
|
||||||
|
case "board":
|
||||||
|
return boardCommand(ctx, args[1:])
|
||||||
|
case "api":
|
||||||
|
return apiCommand(ctx, args[1:])
|
||||||
|
case "licence":
|
||||||
|
return licenceCommand(ctx, args[1:])
|
||||||
|
case "rotate":
|
||||||
|
return rotateCommand(ctx, args[1:])
|
||||||
|
case "ask":
|
||||||
|
return askCommand(ctx, args[1:])
|
||||||
|
case "builds":
|
||||||
|
return buildsCommand(ctx, args[1:])
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||||
|
case "queue":
|
||||||
|
return queueCommand(ctx, args[1:])
|
||||||
|
case "cancel":
|
||||||
|
return cancelCommand(ctx, args[1:])
|
||||||
|
case "clear":
|
||||||
|
return clearCommand(ctx, args[1:])
|
||||||
|
case "rebuild":
|
||||||
|
return rebuildCommand(ctx, args[1:])
|
||||||
|
case "replay":
|
||||||
|
return replayCommand(ctx, args[1:])
|
||||||
|
case "kill":
|
||||||
|
return killCommand(ctx, args[1:])
|
||||||
|
case "pause", "resume":
|
||||||
|
return pauseCommand(ctx, args[0], args[1:])
|
||||||
|
case "collection":
|
||||||
|
return collectionCommand(ctx, args[1:])
|
||||||
|
case "plans":
|
||||||
|
return plansCommand(ctx, args[1:])
|
||||||
|
case "pin":
|
||||||
|
return pinCommand(ctx, args[1:], true)
|
||||||
|
case "unpin":
|
||||||
|
return pinCommand(ctx, args[1:], false)
|
||||||
|
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
|
||||||
|
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
|
||||||
|
// own schema is not a special case. `migrate` remains the word a person types.
|
||||||
|
case "prepare", "migrate":
|
||||||
|
return migrate(ctx)
|
||||||
|
case "node":
|
||||||
|
return nodeCommand(ctx, args[1:])
|
||||||
|
case "token":
|
||||||
|
return tokenCommand(ctx, args[1:])
|
||||||
|
case "identity":
|
||||||
|
return identityCommand(ctx, args[1:])
|
||||||
|
case "broker":
|
||||||
|
return brokerCommand(ctx, args[1:])
|
||||||
|
case "serve":
|
||||||
|
return serve(ctx)
|
||||||
|
case "upgrade":
|
||||||
|
return upgradeCommand(ctx, args[1:])
|
||||||
|
case "declare":
|
||||||
|
return declare(ctx, args[1:])
|
||||||
|
case "overlay":
|
||||||
|
return overlayCommand(ctx, args[1:])
|
||||||
|
case "module":
|
||||||
|
return moduleCommand(ctx, args[1:])
|
||||||
|
case "assign", "unassign":
|
||||||
|
return assignCommand(ctx, args[0], args[1:])
|
||||||
|
case "take":
|
||||||
|
return takeCommand(ctx, args[1:])
|
||||||
|
case "converge":
|
||||||
|
return convergeCommand(ctx, args[1:])
|
||||||
|
case "adopt":
|
||||||
|
return adoptCommand(ctx, args[1:])
|
||||||
|
case "settings":
|
||||||
|
return settingsCommand(ctx, args[1:])
|
||||||
|
case "secret":
|
||||||
|
return secretCommand(ctx, args[1:])
|
||||||
|
case "operator":
|
||||||
|
return operatorCommand(ctx, args[1:])
|
||||||
|
case "plan":
|
||||||
|
return planCommand(ctx, args[1:])
|
||||||
|
case "push":
|
||||||
|
return pushCommand(ctx, args[1:])
|
||||||
|
case "rollout":
|
||||||
|
return rolloutCommand(ctx, args[1:])
|
||||||
|
case "seats":
|
||||||
|
return seatsCommand(ctx, args[1:])
|
||||||
|
case "seat":
|
||||||
|
return seatCommand(ctx, args[1:])
|
||||||
|
case "status":
|
||||||
|
return statusCommand(ctx, args[1:])
|
||||||
|
case "version":
|
||||||
|
fmt.Println(version)
|
||||||
|
return nil
|
||||||
|
case "help", "-h", "--help":
|
||||||
|
usage()
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
usage()
|
||||||
|
return fmt.Errorf("%q is not a command", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func usage() {
|
||||||
|
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||||
|
|
||||||
|
migrate bring each context's schema up to date
|
||||||
|
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
||||||
|
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||||
|
node list the nodes this mesh knows about
|
||||||
|
node show <name> what one machine reported it can do, and why
|
||||||
|
node public-domain <name> the domain it composes its routed names under
|
||||||
|
node public-domain <name> <d> ...set it to d
|
||||||
|
node public-domain <name> --clear ...it faces the outside no longer
|
||||||
|
node networks <name> the networks it routes for what it hosts
|
||||||
|
node networks <name> <cidr>... ...set them; its filter forwards these too
|
||||||
|
node networks <name> --clear ...only the container runtime's own
|
||||||
|
token issue --node <name> a one-time right to join, for an existing record
|
||||||
|
token issue --new <name> create the record and issue for it
|
||||||
|
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||||
|
identity show this control plane's signing key
|
||||||
|
broker show where the broker is, and what to expect there
|
||||||
|
serve consume what nodes say, and answer
|
||||||
|
declare <node> <file> send a node a signed declaration
|
||||||
|
overlay place <node> [flags] say where a node is and how it is reached
|
||||||
|
overlay show the private network, as the mesh computes it
|
||||||
|
module add <file> register a module from its manifest
|
||||||
|
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
||||||
|
module list what modules this mesh knows about
|
||||||
|
module moved <name> <commit> the source has a newer commit than the mesh built
|
||||||
|
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
||||||
|
module forget <name> --and-what-it-holds ...and discard its settings, secrets and ports too
|
||||||
|
module issue <name> --node <m> a broker account for a module, scoped to its emits and consumes
|
||||||
|
upgrade <name> what happens when this module's current version moves
|
||||||
|
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||||
|
upgrade <name> record ...record that they are behind, and send nothing
|
||||||
|
status [--json] what is wrong, what is quiet, and what is out of date
|
||||||
|
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||||
|
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||||
|
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||||
|
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||||
|
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||||
|
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||||
|
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||||
|
unassign <node> <module>... take them off
|
||||||
|
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||||
|
what it declares; --yes <digest> cuts it over as previewed
|
||||||
|
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||||
|
adopt <node> return a converged node to adopted; what was taken stays taken
|
||||||
|
settings set <module> <file> what a module's config should say, for the whole mesh
|
||||||
|
settings set <module> <file> --node <n> ...or for one machine
|
||||||
|
settings clear <module> [--node <n>] take a layer away
|
||||||
|
secret accept <node> <module> <name> carry a value the mesh did not make and cannot invent
|
||||||
|
secret accept ... --from <file> ...read it from a file rather than being asked
|
||||||
|
secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
|
||||||
|
break-glass: open the operator-sealed copy, to a 0600 file
|
||||||
|
secret export [--out <file>] every operator-sealed copy, ciphertext — keep it with the key
|
||||||
|
operator key make [--out <file>] make the operator's sealing key, off the mesh; private half to the file only
|
||||||
|
operator key set <public> [--replace] tell the mesh which operator key to seal to
|
||||||
|
operator key show the operator key, and what it can recover
|
||||||
|
build <repository> [--ref R] have a build machine build it, and record what came out
|
||||||
|
build --behind build every module the mesh holds older than its source
|
||||||
|
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||||
|
builds [<module>] what has been built lately, and what came of it
|
||||||
|
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||||
|
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||||
|
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||||
|
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||||
|
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||||
|
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||||
|
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||||
|
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||||
|
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||||
|
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||||
|
delivered as the builder module's broker secret (module add it first)
|
||||||
|
licence add|list|use|key model access, under the name a person calls it
|
||||||
|
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||||
|
licence refresh <name> mint a new access token and seal it to every holder
|
||||||
|
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||||
|
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||||
|
pin <node> <provision> <from-node> <module>
|
||||||
|
which provider this one gets a provision from: the module, and its node
|
||||||
|
unpin <node> <provision> put that question back
|
||||||
|
plan <node> [--files|--json] what that node would run, and why
|
||||||
|
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||||
|
version what this binary is
|
||||||
|
|
||||||
|
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||||
|
`+store.Variable("<context>")+` — or `+store.FileVariable("<context>")+`, naming a file that holds
|
||||||
|
the same thing and keeps the password out of the environment. This process holds:
|
||||||
|
|
||||||
|
`)
|
||||||
|
for _, c := range held {
|
||||||
|
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
||||||
|
c.name, store.Database(c.name), store.Variable(c.name))
|
||||||
|
}
|
||||||
|
fmt.Fprintln(os.Stderr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseAround reads flags that may sit before, after or between positional arguments.
|
||||||
|
//
|
||||||
|
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
|
||||||
|
// parses no flags at all and silently ignores every one of them. The host learned this the same
|
||||||
|
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
|
||||||
|
// keeps naming, and it looks exactly like success.
|
||||||
|
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||||
|
var positionals []string
|
||||||
|
rest := args
|
||||||
|
for {
|
||||||
|
if err := set.Parse(rest); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
rest = set.Args()
|
||||||
|
if len(rest) == 0 {
|
||||||
|
return positionals, nil
|
||||||
|
}
|
||||||
|
positionals = append(positionals, rest[0])
|
||||||
|
rest = rest[1:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Built is the daemon hearing a build's outcome on the bus — its own asking, an announcement's, or
|
||||||
|
// a tool's that did not wait (novox/hq issue 176) — and taking it in: recorded, and the module
|
||||||
|
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||||
|
// became of a build nobody was watching.
|
||||||
|
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||||
|
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||||
|
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||||
|
// before anyone had reviewed it.
|
||||||
|
if result.DryRun {
|
||||||
|
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||||
|
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||||
|
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||||
|
asked, _ := link.BuildAskedAt(result.ID)
|
||||||
|
switch {
|
||||||
|
case err != nil && result.Failed != "":
|
||||||
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
|
if result.Module != "" {
|
||||||
|
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||||
|
} else {
|
||||||
|
planFailedBuild(ctx, b.open, result)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
case errors.Is(err, inventory.ErrSuperseded):
|
||||||
|
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||||
|
// before that later request is answered by it; one that asked after it ignores this.
|
||||||
|
fmt.Printf("%s: %v\n", result.ID, err)
|
||||||
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
|
return nil
|
||||||
|
case err != nil:
|
||||||
|
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||||
|
if manifest.Module != "" {
|
||||||
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||||
|
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||||
|
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||||
|
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ecdh"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A mesh a command can be run against.
|
||||||
|
//
|
||||||
|
// The commands here were tested through the pieces they call and never through themselves, so
|
||||||
|
// three faults that only exist where the pieces meet — an assignment reported as fine while it
|
||||||
|
// blocked other machines, a read-shaped invocation that wrote, a JSON interface that stopped
|
||||||
|
// emitting JSON — were invisible to every test in this package. This raises the real stores and
|
||||||
|
// calls the real functions.
|
||||||
|
|
||||||
|
// aMesh is two placed, capable machines on a private network, with nothing assigned but the
|
||||||
|
// network itself.
|
||||||
|
//
|
||||||
|
// `anchor` is the hub. That is not decoration: a mesh whose hub cannot be resolved has no private
|
||||||
|
// network at all, which is how one machine's problem reaches every other.
|
||||||
|
func aMesh(t *testing.T) *stores {
|
||||||
|
t.Helper()
|
||||||
|
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
|
||||||
|
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||||
|
|
||||||
|
open, err := openStores(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(open.Close)
|
||||||
|
for _, m := range provided {
|
||||||
|
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, name := range []string{"anchor", "laptop"} {
|
||||||
|
record, err := open.inventory.AddNode(t.Context(), name)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetPlace(t.Context(), name, name+".example:51820", "here",
|
||||||
|
name == "anchor", fmt.Sprintf("10.77.0.%d", i+1)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||||
|
{"name": "container-runtime", "present": true},
|
||||||
|
{"name": "wireguard", "present": true},
|
||||||
|
{"name": "systemd", "present": true},
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var profile map[string]any
|
||||||
|
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return open
|
||||||
|
}
|
||||||
|
|
||||||
|
// aPublicKey is a key a machine could have reported. Its private half is thrown away: nothing here
|
||||||
|
// opens anything, it only needs the mesh to believe a machine has a key.
|
||||||
|
func aPublicKey(t *testing.T) string {
|
||||||
|
t.Helper()
|
||||||
|
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
|
||||||
|
}
|
||||||
|
|
||||||
|
// register puts a manifest in the catalogue.
|
||||||
|
func register(t *testing.T, open *stores, m catalogue.Manifest) {
|
||||||
|
t.Helper()
|
||||||
|
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rivals are two modules that cannot share a machine, which is the shortest way to make a node's
|
||||||
|
// own set of assignments incoherent using nothing but commands a person has.
|
||||||
|
func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||||
|
claim := []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}
|
||||||
|
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||||
|
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||||
|
}
|
||||||
@@ -0,0 +1,791 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// the catalogue: what exists, what is assigned, and how it is configured.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
// provided is what comes with the control plane rather than from a repository.
|
||||||
|
//
|
||||||
|
// WireGuard, the names, and the domain module over both. The first two are here because the code
|
||||||
|
// that works out their files is here:
|
||||||
|
// a peer list is derived from every machine at once, so it cannot be written in a manifest, and
|
||||||
|
// whatever computes it has to live wherever the whole picture is.
|
||||||
|
//
|
||||||
|
// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent
|
||||||
|
// from a machine nobody gave it to.
|
||||||
|
func providedModules() []catalogue.Manifest {
|
||||||
|
var out []catalogue.Manifest
|
||||||
|
for _, raw := range []map[string]any{
|
||||||
|
// **Two used to be here and are gone**: one wrote the mesh's names into a hosts file, the
|
||||||
|
// other wrote the same machines as wildcards for a resolver to read. Neither ran software
|
||||||
|
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||||
|
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||||
|
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||||
|
overlay.Manifest(), overlay.DomainManifest(),
|
||||||
|
} {
|
||||||
|
var m catalogue.Manifest
|
||||||
|
b, _ := json.Marshal(raw)
|
||||||
|
_ = json.Unmarshal(b, &m)
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
var provided = providedModules()
|
||||||
|
|
||||||
|
func moduleCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("module add <file>, module check <file>..., module list, or module forget <name>")
|
||||||
|
}
|
||||||
|
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||||
|
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||||
|
if args[0] == "check" {
|
||||||
|
var paths []string
|
||||||
|
for _, a := range args[1:] {
|
||||||
|
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||||
|
under, err := manifestsUnder(a)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
paths = append(paths, under...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
paths = append(paths, a)
|
||||||
|
}
|
||||||
|
return moduleCheck(paths, os.Stdout)
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "add":
|
||||||
|
set := flag.NewFlagSet("module add", flag.ContinueOnError)
|
||||||
|
repo := set.String("source", "", "where this module comes from")
|
||||||
|
ref := set.String("ref", "", "the branch followed there")
|
||||||
|
commit := set.String("commit", "", "the commit this manifest was read at")
|
||||||
|
// **Where inside the repository the module is** (novox/hq ADR 0069). A module is a
|
||||||
|
// repository *and* a directory, and a record that carries only the repository names a
|
||||||
|
// module.json at its root — so every later build of it looks in the wrong place and fails
|
||||||
|
// with "no module.json at its root". Nine modules on this mesh were registered that way
|
||||||
|
// and none of them could be rebuilt (2026-09-28).
|
||||||
|
path := set.String("path", "", "the module's directory inside that repository")
|
||||||
|
self := set.Bool("self", false, "the source is a path on the forge holding the git seat")
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("module add <manifest.json> [--source <repo> [--self] [--path P] " +
|
||||||
|
"--ref <branch> --commit <sha>]")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(positionals[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := namesNoInstallation(m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.RegisterModule(ctx, m, from); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s registered", m.Module)
|
||||||
|
if *commit != "" {
|
||||||
|
fmt.Printf(" from %s", short(*commit))
|
||||||
|
}
|
||||||
|
if *repo != "" && *path == "" {
|
||||||
|
// Said, not refused: a module really at the root is the ordinary case for a repository
|
||||||
|
// of its own. But a repository holding many modules and a record naming none of them is
|
||||||
|
// a module nothing can rebuild, and the person adding it is the one who knows which.
|
||||||
|
fmt.Printf("\n no directory inside %s, so it is built from that repository's root — "+
|
||||||
|
"`--path` if the module lives in a directory there", *repo)
|
||||||
|
}
|
||||||
|
if len(m.Provides) > 0 {
|
||||||
|
fmt.Printf(", providing %s", describeOffers(m.Provides))
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
fmt.Printf(" claims %s, one per %s\n", c.Name, c.At())
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "list":
|
||||||
|
// The catalogue: what exists, where it came from, whether it is current, and who runs it.
|
||||||
|
// The provenance was recorded from the first build and nothing showed it, which made
|
||||||
|
// "is this current?" a question you could only answer by reading the database.
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||||
|
// is, where it runs, whether it is current, and what it says of itself.
|
||||||
|
if len(args) > 1 && args[1] == "--json" {
|
||||||
|
type listed struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Version string `json:"version"`
|
||||||
|
Built string `json:"built,omitempty"`
|
||||||
|
Head string `json:"head,omitempty"`
|
||||||
|
Current bool `json:"current"`
|
||||||
|
Provided bool `json:"provided,omitempty"`
|
||||||
|
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||||
|
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||||
|
// what the console checks the bus's answers against.
|
||||||
|
Tools bool `json:"tools"`
|
||||||
|
On []string `json:"on"`
|
||||||
|
Provides []string `json:"provides,omitempty"`
|
||||||
|
Requires []string `json:"requires,omitempty"`
|
||||||
|
Claims []string `json:"claims,omitempty"`
|
||||||
|
Capabilities []string `json:"capabilities,omitempty"`
|
||||||
|
}
|
||||||
|
out := make([]listed, 0, len(entries))
|
||||||
|
for _, e := range entries {
|
||||||
|
m := e.Manifest
|
||||||
|
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||||
|
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||||
|
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||||
|
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||||
|
}
|
||||||
|
out = append(out, l)
|
||||||
|
}
|
||||||
|
return printJSON(out)
|
||||||
|
}
|
||||||
|
if len(entries) == 0 {
|
||||||
|
fmt.Println("this mesh knows about no modules yet")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var stale int
|
||||||
|
for _, e := range entries {
|
||||||
|
m := e.Manifest
|
||||||
|
fmt.Printf("%-18s %-8s", m.Module, m.Version)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case e.Provided:
|
||||||
|
fmt.Printf(" %-22s", "with the control plane")
|
||||||
|
case e.Source.Repository == "":
|
||||||
|
// Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and
|
||||||
|
// worth saying, because nothing can rebuild it.
|
||||||
|
fmt.Printf(" %-22s", "handed over")
|
||||||
|
case !e.Source.Current():
|
||||||
|
stale++
|
||||||
|
fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head))
|
||||||
|
default:
|
||||||
|
fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(e.On) > 0 {
|
||||||
|
fmt.Printf(" on %s", strings.Join(e.On, ", "))
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" on nothing")
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
var says []string
|
||||||
|
if len(m.Provides) > 0 {
|
||||||
|
says = append(says, "provides "+describeOffers(m.Provides))
|
||||||
|
}
|
||||||
|
if len(m.Requires) > 0 {
|
||||||
|
says = append(says, "requires "+strings.Join(m.Requires, ", "))
|
||||||
|
}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
says = append(says, "claims "+c.At()+"/"+c.Name)
|
||||||
|
}
|
||||||
|
if len(m.Capabilities) > 0 {
|
||||||
|
says = append(says, "needs "+strings.Join(m.Capabilities, ", "))
|
||||||
|
}
|
||||||
|
if len(says) > 0 {
|
||||||
|
fmt.Printf(" %s\n", strings.Join(says, " · "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if stale > 0 {
|
||||||
|
fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "moved":
|
||||||
|
if len(args) != 3 {
|
||||||
|
return errors.New("module moved <name> <commit> — the source has a newer commit")
|
||||||
|
}
|
||||||
|
if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
from, err := inv.SourceOf(ctx, args[1])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if from.Current() {
|
||||||
|
fmt.Printf("%s is current at %s\n", args[1], short(from.Head))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
|
||||||
|
args[1], short(from.BuiltFrom), short(from.Head))
|
||||||
|
fmt.Printf(" run `build %s` to catch up\n", from.Repository)
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "forget":
|
||||||
|
// **What goes with it is said before it goes** (novox/hq 04-ISSUES/017). The settings, the
|
||||||
|
// module's own secrets and the ports the mesh chose all cascade off the module row, so
|
||||||
|
// `forget` used to destroy them and report "forgotten" — an action succeeding into a state
|
||||||
|
// its own verify would reject, and a sealed secret is not recoverable afterwards.
|
||||||
|
set := flag.NewFlagSet("module forget", flag.ContinueOnError)
|
||||||
|
andHeld := set.Bool("and-what-it-holds", false,
|
||||||
|
"discard its settings, its own secrets and its ports along with it")
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("module forget <name> [--and-what-it-holds]")
|
||||||
|
}
|
||||||
|
if !*andHeld {
|
||||||
|
if err := inv.ForgetModule(ctx, positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s forgotten\n", positionals[0])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
held, err := inv.DiscardModule(ctx, positionals[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s forgotten\n", positionals[0])
|
||||||
|
for _, line := range held.Lines() {
|
||||||
|
// Said after the fact as well as before it: this is the only record that these
|
||||||
|
// existed, and the next person to ask why the module came back empty reads it here.
|
||||||
|
fmt.Printf(" discarded%s\n", strings.TrimPrefix(line, " "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "issue":
|
||||||
|
// A module's broker account, scoped by its emits and consumes (novox/hq ADR 0043) and
|
||||||
|
// sealed to the machine that will run it — the generic case the builder was the first of.
|
||||||
|
set := flag.NewFlagSet("module issue", flag.ContinueOnError)
|
||||||
|
forNode := set.String("node", "",
|
||||||
|
"the machine that will run it, so the credential is delivered instead of printed")
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("module issue <module> --node <machine>")
|
||||||
|
}
|
||||||
|
module := positionals[0]
|
||||||
|
if *forNode == "" {
|
||||||
|
return errors.New("module issue needs --node: a module's account is sealed to the " +
|
||||||
|
"machine that runs it, and the mesh cannot read it back to print")
|
||||||
|
}
|
||||||
|
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
m, ok := shelf[module]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("this mesh knows no module %q; `module add` it first", module)
|
||||||
|
}
|
||||||
|
if err := mayIssue(m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Which bus this mesh is on. A module gets a credential for exactly one, and the two are
|
||||||
|
// made in entirely different ways: on the bus the mesh runs on today an account is a
|
||||||
|
// management call, and on the bus being built it is a row the next composition writes into
|
||||||
|
// the server's user list (novox/hq design 25 §4).
|
||||||
|
busAddress, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("module has no %q; it has add, check, list, moved, forget and issue", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||||
|
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||||
|
// service manager and the package manager — can only go on, or come off, together.
|
||||||
|
if len(args) < 2 {
|
||||||
|
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
// The act itself is in acts.go, so the command API refuses exactly what this refuses
|
||||||
|
// (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed.
|
||||||
|
act := assign
|
||||||
|
if verb == "unassign" {
|
||||||
|
act = unassign
|
||||||
|
}
|
||||||
|
said, err := act(ctx, open, args[0], args[1:]...)
|
||||||
|
if said != "" {
|
||||||
|
fmt.Println(said)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func settingsCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
set := flag.NewFlagSet("settings", flag.ContinueOnError)
|
||||||
|
node := set.String("node", "", "one machine, rather than the whole mesh")
|
||||||
|
positionals, err := parseAround(set, args[1:])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
where := "the whole mesh"
|
||||||
|
if *node != "" {
|
||||||
|
where = *node
|
||||||
|
}
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "set":
|
||||||
|
if len(positionals) != 2 {
|
||||||
|
return errors.New("settings set <module> <settings.json | {…}> [--node <node>]")
|
||||||
|
}
|
||||||
|
// A file, or the values themselves when they begin with `{` — which is how the mesh's own
|
||||||
|
// `settings` tool passes them, having no file to hand over (novox/hq issue 198).
|
||||||
|
var raw []byte
|
||||||
|
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
|
||||||
|
raw = []byte(positionals[1])
|
||||||
|
} else if raw, err = os.ReadFile(positionals[1]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var values map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &values); err != nil {
|
||||||
|
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
|
||||||
|
}
|
||||||
|
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var keys []string
|
||||||
|
for k := range values {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
|
||||||
|
fmt.Println(" run `push` to send it")
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "clear":
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("settings clear <module> [--node <node>]")
|
||||||
|
}
|
||||||
|
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
||||||
|
return nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
|
||||||
|
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
|
||||||
|
// entirely different things about where the answer has to be.
|
||||||
|
func describeOffers(offers []catalogue.Offer) string {
|
||||||
|
var out []string
|
||||||
|
for _, o := range offers {
|
||||||
|
if o.At() == catalogue.ScopeMesh {
|
||||||
|
out = append(out, o.Name+" (from anywhere in the mesh)")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, o.Name)
|
||||||
|
}
|
||||||
|
return strings.Join(out, ", ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// pinCommand says which node a machine gets a provision from.
|
||||||
|
//
|
||||||
|
// Needed only when more than one could answer, and recordable before that -- a mesh with one
|
||||||
|
// database should not change where an existing machine gets its data the day a second one
|
||||||
|
// arrives.
|
||||||
|
func pinCommand(ctx context.Context, args []string, setting bool) error {
|
||||||
|
if setting && len(args) != 4 {
|
||||||
|
return errors.New("pin <node> <provision> <from-node> <module>")
|
||||||
|
}
|
||||||
|
if !setting && len(args) != 2 {
|
||||||
|
return errors.New("unpin <node> <provision>")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
if !setting {
|
||||||
|
if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// The provider's node may be this same machine: two modules beside the consumer can both
|
||||||
|
// answer a provision, and then the module is the whole question (novox/hq #258).
|
||||||
|
if err := inv.PinProvision(ctx, args[0], args[1], args[2], args[3]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s gets %s from %s/%s\n", args[0], args[1], args[2], args[3])
|
||||||
|
fmt.Printf(" run `push %s` to send it\n", args[0])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// theBrokerSeat is the mesh-scoped seat the broker module claims (novox/hq ADR 0079: a
|
||||||
|
// foundation seat is named after the server it guards).
|
||||||
|
const theBrokerSeat = "mesh-broker"
|
||||||
|
|
||||||
|
// brokerReachableAt is the broker's address as the given node can reach it.
|
||||||
|
//
|
||||||
|
// The genesis address (MESH_BROKER_ADDRESS) is the broker's public endpoint — right for a machine
|
||||||
|
// that can dial it, wrong across the mesh, where the path is the overlay. A node that is ON the
|
||||||
|
// overlay is given the overlay name of the node HOLDING the broker — the one assigned a module
|
||||||
|
// claiming the mesh-broker seat — because that is where the broker is, whatever else the topology
|
||||||
|
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||||
|
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||||
|
//
|
||||||
|
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the networking
|
||||||
|
// module — not "has an address", which is true of every placed machine and says nothing about
|
||||||
|
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||||
|
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||||
|
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||||
|
func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known broker.Broker, node string) (string, error) {
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
// No catalogue yet is genesis, and at genesis the genesis address is the right one.
|
||||||
|
return known.Address, nil
|
||||||
|
}
|
||||||
|
if onNetwork[node] == "" {
|
||||||
|
return known.Address, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where the broker is: the node whose assigned set includes a module claiming the seat.
|
||||||
|
holders := map[string]bool{}
|
||||||
|
for name, m := range shelf {
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if c.Name == theBrokerSeat && c.At() == catalogue.ScopeMesh {
|
||||||
|
holders[name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
brokerAt := ""
|
||||||
|
if len(holders) > 0 {
|
||||||
|
overlays, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, o := range overlays {
|
||||||
|
assigned, err := inv.Assigned(ctx, o.Name)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, a := range assigned {
|
||||||
|
if holders[a] && onNetwork[o.Name] != "" {
|
||||||
|
brokerAt = onNetwork[o.Name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if brokerAt == "" {
|
||||||
|
// Nothing holds the seat (or its node is not on the overlay): the hub, where the
|
||||||
|
// foundation is by convention — and only a hub that is itself on the overlay, or the
|
||||||
|
// name we hand out routes nowhere.
|
||||||
|
overlays, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, o := range overlays {
|
||||||
|
if o.Hub && onNetwork[o.Name] != "" {
|
||||||
|
brokerAt = onNetwork[o.Name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if brokerAt == "" {
|
||||||
|
return known.Address, nil
|
||||||
|
}
|
||||||
|
// A portless genesis address is a working configuration (amqps defaults to 5671), and
|
||||||
|
// silently keeping the public address would disable this whole path — so the port defaults
|
||||||
|
// rather than the fix dissolving (novox/hq issue 059).
|
||||||
|
_, port, err := net.SplitHostPort(known.Address)
|
||||||
|
if err != nil {
|
||||||
|
port = "5671"
|
||||||
|
}
|
||||||
|
return net.JoinHostPort(brokerAt, port), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mayIssue says whether a module can be given a broker account. The account is delivered as the
|
||||||
|
// module's own secret named broker; a module that declares none has nothing to read it with, and
|
||||||
|
// an account nothing reads is an orphan on the bus (novox/hq 04-ISSUES/078). Said before the
|
||||||
|
// account is made, not after.
|
||||||
|
func mayIssue(m catalogue.Manifest) error {
|
||||||
|
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||||
|
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||||
|
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||||
|
//
|
||||||
|
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||||
|
// is minted into the mesh's records and becomes usable at the next composition, so there is no
|
||||||
|
// server to be reachable for this to work. The password travels beside the address rather than inside
|
||||||
|
// it, because the runtime's contract already separates them and a credential embedded in a URL is one
|
||||||
|
// that leaks into every log line that prints a connection. And the module's durable consumer is
|
||||||
|
// derived from what it declared rather than declared by name, so a module cannot ask for delivery of
|
||||||
|
// something it did not say it consumes.
|
||||||
|
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||||
|
node, busAddress string) error {
|
||||||
|
|
||||||
|
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
||||||
|
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
||||||
|
// trust store, so an address alone fails at TLS.
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||||
|
}
|
||||||
|
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||||
|
}
|
||||||
|
|
||||||
|
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||||
|
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
|
||||||
|
// runtime is issued through this same path — and the kind is what a reader of the records sees.
|
||||||
|
func busKindOf(module string) string {
|
||||||
|
if module == catalogue.RuntimeModule {
|
||||||
|
return inventory.BusNodeTools
|
||||||
|
}
|
||||||
|
return inventory.BusModule
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||||
|
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||||
|
// so the move can issue every module against a bus whose address it worked out itself
|
||||||
|
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
|
||||||
|
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||||
|
node, busAddress string, known broker.Broker, reachable, user, password string) error {
|
||||||
|
// The seats this module claims, with the verbs each promises (novox/hq ADR 0159): the runtime
|
||||||
|
// serves a claimed seat's verbs with its tools of the same name, and the bus admits only the
|
||||||
|
// holder's subscription — so the runtime tries each claim and the grant decides. Written here
|
||||||
|
// because this file is the one thing the mesh writes that the runtime reads before it speaks.
|
||||||
|
claims, err := claimsFor(ctx, inv, m)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
Claims []seatClaimed `json:"claims,omitempty"`
|
||||||
|
}{
|
||||||
|
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
|
||||||
|
Node: node, Module: m.Module, User: user, Password: password, Claims: claims,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||||
|
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||||
|
// durable subscription nobody reads.
|
||||||
|
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||||
|
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||||
|
Emits: m.Emits, Consumes: m.Consumes, Serves: m.Tools,
|
||||||
|
}); needed {
|
||||||
|
if busAddress == "" {
|
||||||
|
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||||
|
"`rollout mint` again is harmless)\n", m.Module)
|
||||||
|
} else {
|
||||||
|
js, err := broker.Dial(busAddress)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||||
|
"how %s hears what it consumes: %w", m.Module, err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
if err := js.EnsureConsumer(consumer); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
||||||
|
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
|
||||||
|
fmt.Printf(" and it works once the bus has been told: the user list is composed into the " +
|
||||||
|
"machine holding mesh-broker\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
|
||||||
|
// say to be worth anything later.
|
||||||
|
//
|
||||||
|
// **A module is a repository and a directory inside it** (novox/hq ADR 0069). A record carrying only
|
||||||
|
// the repository names a module.json at its root, so every later build of it looks in the wrong
|
||||||
|
// place — nine modules on this mesh were registered that way and none of them could be rebuilt
|
||||||
|
// (2026-09-28). The directory cannot be checked from here, because the control plane does not clone;
|
||||||
|
// what can be checked is that the record is whole.
|
||||||
|
func whereItComesFrom(repository, ref, commit, path string, self bool) (inventory.Source, error) {
|
||||||
|
// Provenance together or not at all. A source with no commit cannot be compared against
|
||||||
|
// anything, so it would record where the module came from and still never be able to say the
|
||||||
|
// mesh is behind it — which is the one thing recording it is for.
|
||||||
|
if (repository == "") != (commit == "") {
|
||||||
|
return inventory.Source{}, errors.New("--source and --commit go together: a source with " +
|
||||||
|
"no commit cannot be compared against anything, and a commit with no source has " +
|
||||||
|
"nothing to be compared with")
|
||||||
|
}
|
||||||
|
// A directory or a forge with no repository is half a location, and the half it keeps is the
|
||||||
|
// half nothing can be found with.
|
||||||
|
if repository == "" && (path != "" || self) {
|
||||||
|
return inventory.Source{}, errors.New("--path and --self say where inside a source and " +
|
||||||
|
"which forge holds it, so they need --source: without one there is nothing for them " +
|
||||||
|
"to be part of")
|
||||||
|
}
|
||||||
|
from := inventory.Source{Repository: repository, Ref: ref, BuiltFrom: commit, Path: path}
|
||||||
|
if self {
|
||||||
|
if err := onASeat(repository); err != nil {
|
||||||
|
return inventory.Source{}, err
|
||||||
|
}
|
||||||
|
from.Seat = gitSeat
|
||||||
|
}
|
||||||
|
return from, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// namesNoInstallation is the mesh refusing a definition that names an installation, at the moment
|
||||||
|
// it would enter the catalogue (novox/hq ADR 0112, ADR 0155). `module check` says the same thing
|
||||||
|
// earlier, where the author is; this is the last moment the mesh can still say no, and a
|
||||||
|
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
||||||
|
// in the same words. A name meant on purpose is declared with its reason and passes.
|
||||||
|
func namesNoInstallation(m catalogue.Manifest) error {
|
||||||
|
named := catalogue.InstallationProblems(m)
|
||||||
|
if len(named) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%s names an installation, and a definition names none — declare a name meant "+
|
||||||
|
"on purpose under %s with its reason, or take it out:\n - %s",
|
||||||
|
m.Module, catalogue.NamesOnPurpose, strings.Join(named, "\n - "))
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
|
||||||
|
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
|
||||||
|
type seatClaimed struct {
|
||||||
|
Seat string `json:"seat"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Serves []string `json:"serves,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// claimsFor joins a module's claims with the seats' protocols from the mesh's records.
|
||||||
|
func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) ([]seatClaimed, error) {
|
||||||
|
if len(m.Claims) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
seats, err := inv.Seats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
byName := map[string]catalogue.Seat{}
|
||||||
|
for _, s := range seats {
|
||||||
|
byName[s.Name] = s
|
||||||
|
}
|
||||||
|
var out []seatClaimed
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
claimed := seatClaimed{Seat: c.Name, Scope: c.At()}
|
||||||
|
if s, known := byName[c.Name]; known {
|
||||||
|
claimed.Scope = s.Scope
|
||||||
|
// The verbs the runtime serves for the seat: the claim's own when it names them
|
||||||
|
// (ADR 0160), else every verb the seat promises, which its tools then answer.
|
||||||
|
claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)})
|
||||||
|
}
|
||||||
|
out = append(out, claimed)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||||
|
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||||
|
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||||
|
func declaresTools(m catalogue.Manifest) bool {
|
||||||
|
if len(m.Tools) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, b := range m.Bundles {
|
||||||
|
if len(b.Loads) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if len(c.Serves) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `module issue` makes a broker account and delivers it as the module's own secret named broker.
|
||||||
|
// A module that declares none is refused before the account exists, so the bus never carries an
|
||||||
|
// account nothing reads (novox/hq 04-ISSUES/078).
|
||||||
|
func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||||
|
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: catalogue.OwnSecrets{"password": {Path: "/run/password"}}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a module with no broker own secret was issued an account")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"step-ca", "broker", "own-secrets"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}}); err != nil {
|
||||||
|
t.Errorf("a module declaring its broker secret was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,748 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// the private network: who is on it, where, and what they are called.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
|
||||||
|
const DefaultOverlayCIDR = "10.42.0.0/16"
|
||||||
|
|
||||||
|
// overlayRange is the range the mesh allocates node addresses from.
|
||||||
|
//
|
||||||
|
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
|
||||||
|
// found is at that tunnel's address, its peers are at theirs, and every node's address is
|
||||||
|
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
|
||||||
|
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
|
||||||
|
// the range genesis was told, or the default.
|
||||||
|
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||||
|
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if adopted {
|
||||||
|
return tunnel.Range, nil
|
||||||
|
}
|
||||||
|
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
return DefaultOverlayCIDR, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("overlay place <node> [flags], overlay name <address> <name>, or overlay show")
|
||||||
|
}
|
||||||
|
// Answered before anything is opened. A message about which command to use should not need a
|
||||||
|
// database to say so, and needing one turns a redirect into a connection error.
|
||||||
|
if args[0] == "push" {
|
||||||
|
return errors.New("`overlay push` is now `push`, which sends a node its network AND " +
|
||||||
|
"what its assignments resolve to — the two are computed from one picture of the " +
|
||||||
|
"mesh, and sending them separately would let them disagree")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "place":
|
||||||
|
return overlayPlace(ctx, inv, args[1:])
|
||||||
|
case "show":
|
||||||
|
return overlayShow(ctx, open)
|
||||||
|
case "name":
|
||||||
|
return overlayName(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("overlay has no %q; it has place, name and show", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// overlayName is the operator saying which machine a carried address is (novox/hq issue 112),
|
||||||
|
// so the mesh answers for its name until the machine enrols and verifies it.
|
||||||
|
func overlayName(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("overlay name <carried-address> <node-name>")
|
||||||
|
}
|
||||||
|
address, name := args[0], args[1]
|
||||||
|
if err := inv.NamePeer(ctx, address, name); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("the peer at %s is %s until it enrols — the mesh answers for %s.<suffix> from the "+
|
||||||
|
"operator's word, and enrolment under this key must use this name\n", address, name, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New(
|
||||||
|
"overlay place <node> [--endpoint host:port] [--site name] [--hub], or --nothing")
|
||||||
|
}
|
||||||
|
node := args[0]
|
||||||
|
|
||||||
|
set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
|
||||||
|
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
|
||||||
|
site := set.String("site", "", "where this machine physically is, or empty if it roams")
|
||||||
|
hub := set.Bool("hub", false, "this node is the hub every other routes through")
|
||||||
|
nothing := set.Bool("nothing", false,
|
||||||
|
"place it with nothing set: not dialable, no site, not the hub")
|
||||||
|
if err := set.Parse(args[1:]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// **All three are declared together, so saying nothing took all three away.** The sibling of
|
||||||
|
// `node public-domain`: `overlay place anchor` reads like it places the node it names, and it
|
||||||
|
// silently unset the endpoint every other machine dials, the site it is in, and the hub if it
|
||||||
|
// was the hub — every path through it going with them, at the moment somebody was trying to
|
||||||
|
// look at it.
|
||||||
|
//
|
||||||
|
// A placement with nothing set is a real thing to want — a machine that roams and opens every
|
||||||
|
// path itself is exactly that — so it keeps a way to say so, by name.
|
||||||
|
if set.NFlag() == 0 {
|
||||||
|
return fmt.Errorf("overlay place %s was given nothing to place it with, and all three are "+
|
||||||
|
"declared together — it would take away the endpoint other machines dial %s at, its "+
|
||||||
|
"site, and the hub if it is the hub. Say --endpoint/--site/--hub, or --nothing if that "+
|
||||||
|
"is what you meant", node, node)
|
||||||
|
}
|
||||||
|
if *nothing && (*endpoint != "" || *site != "" || *hub) {
|
||||||
|
return fmt.Errorf("give %s a placement or --nothing, not both: they say opposite things "+
|
||||||
|
"and the mesh will not choose between them", node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One hub per mesh, refused rather than last-write-wins: with two flagged, which one the
|
||||||
|
// graph and the broker address pick is order-dependent — the silent-election fault ADR 0007
|
||||||
|
// exists to avoid, one flag over (novox/hq issue 059). Moving the hub is explicit: re-place
|
||||||
|
// the old one without --hub first.
|
||||||
|
if *hub {
|
||||||
|
placed, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, p := range placed {
|
||||||
|
if p.Hub && p.Name != node {
|
||||||
|
return fmt.Errorf("%s is already the hub; a mesh has one. Re-place %s without "+
|
||||||
|
"--hub first if the hub is moving", p.Name, p.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
|
||||||
|
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
|
||||||
|
// have the mesh's interface up where no peer is listening for it.
|
||||||
|
found, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var tunnel inventory.Tunnel
|
||||||
|
adoptsTunnel := false
|
||||||
|
if *hub && found.Adopted {
|
||||||
|
if t, err := inv.TunnelOf(ctx, node); err == nil {
|
||||||
|
tunnel = t
|
||||||
|
placed, _ := inv.Overlays(ctx)
|
||||||
|
for _, o := range placed {
|
||||||
|
if o.Name == node && o.Key == t.PublicKey {
|
||||||
|
adoptsTunnel = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if !errors.Is(err, inventory.ErrNoTunnel) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if adoptsTunnel {
|
||||||
|
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
|
||||||
|
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
|
||||||
|
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
|
||||||
|
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
||||||
|
// election by address prefix fails silently (novox/hq ADR 0007).
|
||||||
|
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
cidr, err := overlayRange(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
address, err := inv.AssignAddress(ctx, found.ID, cidr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("%s is at %s on the overlay\n", node, address)
|
||||||
|
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
|
||||||
|
" `module issue` them again and push (novox/hq issue 059)")
|
||||||
|
switch {
|
||||||
|
case adoptsTunnel:
|
||||||
|
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
|
||||||
|
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
|
||||||
|
len(tunnel.Peers))
|
||||||
|
case *hub:
|
||||||
|
fmt.Println(" the hub — every node not sharing a site routes through it")
|
||||||
|
case *endpoint == "":
|
||||||
|
fmt.Println(" not dialable — it opens every path itself")
|
||||||
|
}
|
||||||
|
if *site != "" {
|
||||||
|
fmt.Printf(" at %s, so it peers directly with anything else there\n", *site)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// network builds the private network over the machines that resolved the module for it.
|
||||||
|
//
|
||||||
|
// Not over every node the mesh knows. **A machine is on the private network because it was given
|
||||||
|
// the module**, and one that was not is absent from every peer list and from the names — which is
|
||||||
|
// the only thing "not on the network" can mean. Until this, having an address was enough, and
|
||||||
|
// there was no way to keep a machine off.
|
||||||
|
//
|
||||||
|
// Every node at once, which is the whole reason this is the control plane's work: a peer list is
|
||||||
|
// derived from all the others, so no node could compute its own.
|
||||||
|
func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||||
|
refused map[string]string) (*overlay.Generator, error) {
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
|
||||||
|
tunnels, err := inv.Tunnels(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
carried, err := inv.CarriedPeers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
nodes := make([]overlay.Node, 0, len(places))
|
||||||
|
for _, p := range places {
|
||||||
|
if !on[p.Name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n := overlay.Node{
|
||||||
|
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||||
|
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||||
|
}
|
||||||
|
if t, takes := tunnels[p.Name]; takes && t.NodeAdopted {
|
||||||
|
// Only an adopted node is told to take the found unit over: on a converged one there
|
||||||
|
// is nothing found to keep, and the host refuses the field. The range and the carried
|
||||||
|
// peers do not depend on the mode; the takeover does.
|
||||||
|
//
|
||||||
|
// **Refused, not composed, when the hub's record disagrees with the tunnel.** A
|
||||||
|
// declaration that stopped the found unit and raised the mesh's interface on another
|
||||||
|
// port or address would leave every peer dark while reporting the tunnel taken — so a
|
||||||
|
// hub placed before it took the tunnel over (or at the wrong port) is named here, and
|
||||||
|
// nothing is sent until it is re-placed.
|
||||||
|
if wrong := disagrees(p, t.Tunnel); wrong != "" {
|
||||||
|
return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+
|
||||||
|
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
||||||
|
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
||||||
|
}
|
||||||
|
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU}
|
||||||
|
}
|
||||||
|
if p.Hub {
|
||||||
|
for _, c := range carried {
|
||||||
|
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes = append(nodes, n)
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
||||||
|
// answers rather than refusing -- Compute would refuse for want of a hub, and reporting
|
||||||
|
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
||||||
|
return overlay.Empty(), nil
|
||||||
|
}
|
||||||
|
cidr, err := overlayRange(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
g, err := overlay.From(nodes, cidr, "")
|
||||||
|
if g != nil {
|
||||||
|
// The artifact store, as this network reaches it. Found rather than configured: the
|
||||||
|
// provider is whichever module offers it, on whichever machine holds that module — and if
|
||||||
|
// nothing does yet (genesis raises the registry before the catalogue knows it), there is
|
||||||
|
// no trust to write and nothing is written (novox/hq ADR 0082).
|
||||||
|
//
|
||||||
|
// Refused rather than composed without it when the question could not be answered: a
|
||||||
|
// declaration missing the trust because a lookup failed is a machine that cannot pull,
|
||||||
|
// delivered by a push that reported success — and nothing recomposes it until the next
|
||||||
|
// push (the shape of novox/hq issues 042/048, reappearing as a race).
|
||||||
|
at, port, found, storeErr := artifactStoreOnNetwork(ctx, inv, on)
|
||||||
|
if storeErr != nil {
|
||||||
|
return nil, fmt.Errorf("finding the artifact store this network reaches: %w", storeErr)
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
g.TrustRegistry(overlay.InternalName(at) + ":" + port)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil && len(refused) > 0 {
|
||||||
|
// The network is missing something, and some machines could not be resolved at all. Those
|
||||||
|
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||||
|
// reporting "no hub" would name a consequence and hide the cause.
|
||||||
|
var who []string
|
||||||
|
for name, why := range refused {
|
||||||
|
who = append(who, fmt.Sprintf(" %s: %s", name, why))
|
||||||
|
}
|
||||||
|
sort.Strings(who)
|
||||||
|
return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+
|
||||||
|
"probably why:\n%s", err, len(refused), strings.Join(who, "\n"))
|
||||||
|
}
|
||||||
|
return g, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// graph is the whole mesh's network, for showing it.
|
||||||
|
func graph(ctx context.Context, open *stores) ([]overlay.Node, overlay.Graph, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
on, refused, err := whoResolves(ctx, open, overlay.Requirement)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
g, err := network(ctx, inv, on, refused)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return g.Nodes(), g.Graph(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// whoResolves is the machines whose resolution answers a requirement, and why the others did not.
|
||||||
|
//
|
||||||
|
// By what a module **provides**, not by its name. WireGuard is one way to have a private network
|
||||||
|
// and there could be others, so a machine is on the network because something it runs provides
|
||||||
|
// one — asking for a particular module by name would be the mistake this whole mechanism exists
|
||||||
|
// to avoid.
|
||||||
|
//
|
||||||
|
// Resolved rather than read from the assignment table, because a module can arrive by being
|
||||||
|
// required by something else, and a machine that needs the private network to do its job is on it
|
||||||
|
// for the same reason as one that was handed it directly.
|
||||||
|
func whoResolves(ctx context.Context, open *stores, requirement string) (
|
||||||
|
map[string]bool, map[string]string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
on := map[string]bool{}
|
||||||
|
// Why a node could not be resolved, kept rather than raised: one broken node must not stop
|
||||||
|
// the rest being described, and whoever is rendering that node will raise it themselves.
|
||||||
|
refused := map[string]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
plan, _, err := planFor(ctx, open, n.Name)
|
||||||
|
switch {
|
||||||
|
case unresolvable(err):
|
||||||
|
refused[n.Name] = err.Error()
|
||||||
|
continue
|
||||||
|
case err != nil:
|
||||||
|
// Not a node that does not resolve — a question that went unanswered. Recording it as a
|
||||||
|
// refusal would take the machine off the private network, and the generator that reads
|
||||||
|
// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
|
||||||
|
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
|
||||||
|
n.Name, requirement, err)
|
||||||
|
}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
for _, offered := range m.Offers() {
|
||||||
|
if offered == requirement {
|
||||||
|
on[n.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return on, refused, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rendering is everything a declaration needs, computed over the whole mesh.
|
||||||
|
func generators(ctx context.Context, open *stores) (
|
||||||
|
map[string]catalogue.Generator, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
on, refused, err := whoResolves(ctx, open, overlay.Addressing)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
net, err := network(ctx, inv, on, refused)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// **One generator now.** Two more used to sit beside it — the names and a resolver's zone
|
||||||
|
// file — as modules that ran nothing. Both are facts a module asks for in its manifest
|
||||||
|
// (`facts:` — catalogue.FactsInto), computed from the same machines this sees: the ones on the
|
||||||
|
// private network, because a name for a machine not on it would resolve to an address nothing
|
||||||
|
// can reach.
|
||||||
|
return map[string]catalogue.Generator{
|
||||||
|
overlay.Name: net,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func overlayShow(ctx context.Context, open *stores) error {
|
||||||
|
nodes, computed, err := graph(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
// Not "this mesh has no nodes", which it said until the network became a module and was
|
||||||
|
// then a lie about the cause: a mesh can have every node it will ever have and nobody on
|
||||||
|
// the private network, because nobody asked for one.
|
||||||
|
fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n",
|
||||||
|
overlay.Name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
|
||||||
|
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
|
||||||
|
// mesh until they enrol — and once one has, it is listed as the node it became.
|
||||||
|
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
carried, err := open.inventory.CarriedPeers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
place := n.Address
|
||||||
|
if place == "" {
|
||||||
|
// Said, not skipped. A node with no place is a node with no network, and it should
|
||||||
|
// be visible here rather than quietly absent from a list of who is on it.
|
||||||
|
place = "no address — run `overlay place`"
|
||||||
|
}
|
||||||
|
fmt.Printf("%-16s %-14s", n.Name, place)
|
||||||
|
switch {
|
||||||
|
case n.Hub && adopted:
|
||||||
|
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
|
||||||
|
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||||
|
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||||
|
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||||
|
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||||
|
case n.Hub:
|
||||||
|
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||||
|
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||||
|
case !n.Reachable():
|
||||||
|
fmt.Print(" not dialable")
|
||||||
|
}
|
||||||
|
if n.Site != "" {
|
||||||
|
fmt.Printf(" at %s", n.Site)
|
||||||
|
}
|
||||||
|
if n.TakesOver != nil && !n.Hub {
|
||||||
|
fmt.Printf(" takes over %s", n.TakesOver.Interface)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
for _, p := range computed[n.Name] {
|
||||||
|
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(carried) > 0 {
|
||||||
|
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
|
||||||
|
for _, c := range carried {
|
||||||
|
state := "not yet enrolled"
|
||||||
|
if c.EnrolledAs != "" {
|
||||||
|
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// disagrees says how a node's placement differs from the tunnel it takes over — its address not
|
||||||
|
// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree.
|
||||||
|
func disagrees(p inventory.Overlay, t inventory.Tunnel) string {
|
||||||
|
var wrong []string
|
||||||
|
want := t.Address
|
||||||
|
if i := strings.Index(want, "/"); i >= 0 {
|
||||||
|
want = want[:i]
|
||||||
|
}
|
||||||
|
if p.Address != want {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want))
|
||||||
|
}
|
||||||
|
if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) {
|
||||||
|
wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port))
|
||||||
|
}
|
||||||
|
return strings.Join(wrong, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNothing(s string) string {
|
||||||
|
if s == "" {
|
||||||
|
return "unset"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// portOfEndpoint is the port in host:port, or empty.
|
||||||
|
func portOfEndpoint(endpoint string) string {
|
||||||
|
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||||
|
return endpoint[i+1:]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// SilentFor is how long a node may be quiet before the mesh says so.
|
||||||
|
//
|
||||||
|
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
||||||
|
// is not the point — being able to say "out of touch" at all is, and nothing could before.
|
||||||
|
const SilentFor = 3 * time.Minute
|
||||||
|
|
||||||
|
// whereEveryoneIs is each machine's name on the private network, for the ones on it.
|
||||||
|
//
|
||||||
|
// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every
|
||||||
|
// other path here answers a question about one node by resolving the others; this one is called
|
||||||
|
// *from* that path, so doing the same would not terminate — which it did not, for two minutes,
|
||||||
|
// until it was run.
|
||||||
|
//
|
||||||
|
// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the
|
||||||
|
// private network depends on what it was assigned and what that requires, both of which are local
|
||||||
|
// facts. What it takes *from* other machines does not change the answer.
|
||||||
|
//
|
||||||
|
// The distinction that matters is kept: a machine absent from the network module's own view is
|
||||||
|
// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the
|
||||||
|
// network became something a machine is given.
|
||||||
|
func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||||
|
|
||||||
|
if shelf == nil {
|
||||||
|
// Refused rather than answered. Being on the private network is a conclusion about what a
|
||||||
|
// node resolves to, so with no catalogue nothing resolves and the honest answer is
|
||||||
|
// "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused
|
||||||
|
// every certificate the mesh was asked for while saying the machine was on no network.
|
||||||
|
return nil, errors.New(
|
||||||
|
"asked where everyone is without the catalogue, which cannot be answered")
|
||||||
|
}
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, p := range places {
|
||||||
|
out[p.Name] = overlay.InternalName(p.Name)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// onTheNetwork is every placed machine that resolves the private network — has an address AND
|
||||||
|
// runs what puts it there. "Has an address" alone was true of every placed machine and told you
|
||||||
|
// nothing about whether anything could reach it; a name written for such a machine resolves to
|
||||||
|
// an address that does not answer, and a connection to it hangs (novox/hq issue 079).
|
||||||
|
func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) ([]inventory.Overlay, error) {
|
||||||
|
places, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// **With the seat holders on record**, or a machine running the next holder of a seat beside
|
||||||
|
// the current one resolves as two holders, is refused, and drops out of the map — taking the
|
||||||
|
// address every other machine composes for what it offers (novox/hq ADR 0131). Found live:
|
||||||
|
// the control node vanished from the private network the moment the new bus was assigned
|
||||||
|
// beside the old one.
|
||||||
|
holdings, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []inventory.Overlay
|
||||||
|
for _, p := range places {
|
||||||
|
if p.Address == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, p.Name)
|
||||||
|
if err != nil || len(assigned) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
caps, _ := inv.ProfileOf(ctx, p.Name)
|
||||||
|
got, err := catalogue.Resolve(shelf, assigned,
|
||||||
|
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
|
||||||
|
catalogue.World{Unchecked: true, Holdings: holdings})
|
||||||
|
if err != nil {
|
||||||
|
// Said, not skipped in silence: a machine dropped here loses its address, and every
|
||||||
|
// plan that names it fails in another module's words (novox/hq issue 188).
|
||||||
|
fmt.Fprintf(os.Stderr, "%s is not counted as on the network: it does not resolve: %v\n", p.Name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range got.Modules {
|
||||||
|
for _, offered := range m.Offers() {
|
||||||
|
if offered == overlay.Requirement {
|
||||||
|
out = append(out, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// onThePrivateNetwork is every node's address on the private network, sorted — the same set
|
||||||
|
// the names and the resolver mean by it (onTheNetwork), so a rule saying "from the mesh" admits
|
||||||
|
// exactly the machines the mesh names.
|
||||||
|
//
|
||||||
|
// A node with no address is left out rather than rendered as an empty source: an empty entry in a
|
||||||
|
// source set is a syntax error in the rule file, and a rule file that does not load leaves the
|
||||||
|
// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule,
|
||||||
|
// because nothing reports it.
|
||||||
|
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) ([]string, error) {
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, p := range places {
|
||||||
|
if strings.TrimSpace(p.Address) != "" {
|
||||||
|
out = append(out, p.Address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// namesInTheMesh is every machine's internal name and the address behind it — every machine
|
||||||
|
// that is on the private network, the same set the resolver means by that.
|
||||||
|
//
|
||||||
|
// A machine that is not has no name: writing one that resolves to nothing is worse than not
|
||||||
|
// writing it, because a connection to an address that does not answer hangs where a name that
|
||||||
|
// does not resolve fails at once and says so. A machine placed on the overlay but not running
|
||||||
|
// the module that puts it there is exactly that (novox/hq issue 079).
|
||||||
|
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||||
|
places, err := onTheNetwork(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := map[string]string{}
|
||||||
|
for _, p := range places {
|
||||||
|
out[overlay.InternalName(p.Name)] = p.Address
|
||||||
|
}
|
||||||
|
// And the carried peers the operator has named (novox/hq issue 112): machines the
|
||||||
|
// predecessor's resolver answers for and the mesh routes to, known by name on the operator's
|
||||||
|
// word until they enrol — at which point enrolment verifies the name and the node's own
|
||||||
|
// entry takes over above. A name the predecessor answers for must keep resolving until the
|
||||||
|
// machine behind it is a node; without these, taking the resolver silences three machines.
|
||||||
|
carried, err := inv.CarriedPeers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, p := range carried {
|
||||||
|
if p.Named == "" || p.EnrolledAs != "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, taken := out[overlay.InternalName(p.Named)]; taken {
|
||||||
|
continue // a node of the mesh owns the name; the stale statement loses
|
||||||
|
}
|
||||||
|
out[overlay.InternalName(p.Named)] = p.Address
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
|
||||||
|
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
|
||||||
|
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
|
||||||
|
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
|
||||||
|
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
|
||||||
|
//
|
||||||
|
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
|
||||||
|
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
|
||||||
|
// and nothing recomposed the machine until the next push. "No store" must mean the mesh has none,
|
||||||
|
// not that the question went unanswered.
|
||||||
|
func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
on map[string]bool) (node, port string, found bool, err error) {
|
||||||
|
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
|
||||||
|
}
|
||||||
|
providers := map[string]catalogue.Manifest{}
|
||||||
|
for name, m := range shelf {
|
||||||
|
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
|
||||||
|
providers[name] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(providers) == 0 {
|
||||||
|
return "", "", false, nil
|
||||||
|
}
|
||||||
|
// In a stated order, so two machines offering it would always answer the same one.
|
||||||
|
machines := make([]string, 0, len(on))
|
||||||
|
for machine := range on {
|
||||||
|
machines = append(machines, machine)
|
||||||
|
}
|
||||||
|
sort.Strings(machines)
|
||||||
|
for _, machine := range machines {
|
||||||
|
assigned, err := inv.Assigned(ctx, machine)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
|
||||||
|
}
|
||||||
|
for _, a := range assigned {
|
||||||
|
m, offers := providers[a]
|
||||||
|
if !offers {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||||
|
}
|
||||||
|
if p, ok := serves["port"]; ok {
|
||||||
|
return machine, fmt.Sprintf("%v", p), true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", "", false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
|
||||||
|
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
|
||||||
|
// node that holds the store itself, and "" for any other. The address composed into every
|
||||||
|
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
|
||||||
|
//
|
||||||
|
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
|
||||||
|
// of those is built and pushed to a node that is on no network, and the store is on that same
|
||||||
|
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
|
||||||
|
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
|
||||||
|
// address genesis itself reaches the store by.
|
||||||
|
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
|
||||||
|
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
|
||||||
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
on := map[string]bool{}
|
||||||
|
for name := range onNetwork {
|
||||||
|
on[name] = true
|
||||||
|
}
|
||||||
|
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if found {
|
||||||
|
return overlay.InternalName(node) + ":" + port, nil
|
||||||
|
}
|
||||||
|
holder, port, found, err := artifactStoreHolder(ctx, inv)
|
||||||
|
if err != nil || !found || holder != forNode {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "127.0.0.1:" + port, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
|
||||||
|
// off the network, and the port that node put it on.
|
||||||
|
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, err
|
||||||
|
}
|
||||||
|
all := map[string]bool{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
all[n.Name] = true
|
||||||
|
}
|
||||||
|
return artifactStoreOnNetwork(ctx, inv, all)
|
||||||
|
}
|
||||||
@@ -0,0 +1,337 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// placementOf is what the mesh holds about where one node is.
|
||||||
|
func placementOf(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string) inventory.Overlay {
|
||||||
|
t.Helper()
|
||||||
|
placed, err := inv.Overlays(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, one := range placed {
|
||||||
|
if one.Name == name {
|
||||||
|
return one
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("%s is not placed at all", name)
|
||||||
|
return inventory.Overlay{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The sibling of `node public-domain`, and the worse one: a placement is three facts declared
|
||||||
|
// together, so an invocation that said none of them took all three away — the endpoint every other
|
||||||
|
// machine dials, the site, and the hub. A mesh whose hub was placed that way has no paths left.
|
||||||
|
func TestPlacingANodeWithNothingSaidDoesNotUnplaceIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := overlayPlace(ctx, open.inventory,
|
||||||
|
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := overlayPlace(ctx, open.inventory, []string{"anchor"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("saying nothing unplaced the node instead of being refused")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "--nothing") {
|
||||||
|
t.Errorf("the refusal does not say how to mean it: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
held := placementOf(t, ctx, open.inventory, "anchor")
|
||||||
|
if held.Endpoint != "198.51.100.10:51820" || held.Site != "hosting" || !held.Hub {
|
||||||
|
t.Fatalf("the placement was taken away by an invocation that was refused: %+v", held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Placing a machine with nothing set is a real thing to want — one that roams and opens every path
|
||||||
|
// itself is exactly that — so it keeps a way to be said, by name.
|
||||||
|
func TestPlacingANodeWithNothingIsAskedForByName(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := overlayPlace(ctx, open.inventory,
|
||||||
|
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := overlayPlace(ctx, open.inventory, []string{"anchor", "--nothing"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
held := placementOf(t, ctx, open.inventory, "anchor")
|
||||||
|
if held.Endpoint != "" || held.Site != "" || held.Hub {
|
||||||
|
t.Fatalf("--nothing did not place it with nothing: %+v", held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Both at once cannot be meant, so neither silently wins.
|
||||||
|
func TestAPlacementAndNothingTogetherIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := overlayPlace(ctx, open.inventory,
|
||||||
|
[]string{"anchor", "--site", "hosting", "--nothing"}); err == nil {
|
||||||
|
t.Fatal("a placement and --nothing together was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine is named for the others only while it is on the private network — placed AND running
|
||||||
|
// what puts it there — the same set the resolver means by "on the private network". A machine
|
||||||
|
// that has an address and no networking is not named: a name resolving to an address that does
|
||||||
|
// not answer hangs where an unknown name fails at once (novox/hq issue 079).
|
||||||
|
func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
|
||||||
|
open := aMesh(t) // two placed machines, both assigned what puts them on the private network
|
||||||
|
ctx := t.Context()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names, err := namesInTheMesh(ctx, open.inventory, shelf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if names["anchor.internal"] != "10.77.0.1" || names["laptop.internal"] != "10.77.0.2" {
|
||||||
|
t.Fatalf("two machines on the network are not both named: %v", names)
|
||||||
|
}
|
||||||
|
// The laptop keeps its place and its address, and stops running the network.
|
||||||
|
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
names, err = namesInTheMesh(ctx, open.inventory, shelf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, still := names["laptop.internal"]; still || names["anchor.internal"] != "10.77.0.1" {
|
||||||
|
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
|
||||||
|
// the tunnel the hub holds — never stored anywhere else.
|
||||||
|
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
||||||
|
|
||||||
|
before, err := overlayRange(ctx, inv)
|
||||||
|
if err != nil || before != "10.99.0.0/16" {
|
||||||
|
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
|
||||||
|
// tunnel's key, and presenting the tunnel.
|
||||||
|
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hub, err := inv.NodeByName(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const key = "THE-TUNNELS-KEY========================="
|
||||||
|
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||||
|
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||||
|
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
|
||||||
|
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
after, err := overlayRange(ctx, inv)
|
||||||
|
if err != nil || after != "192.0.2.0/24" {
|
||||||
|
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
|
||||||
|
// the tunnel's port.
|
||||||
|
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "51900") {
|
||||||
|
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
|
||||||
|
}
|
||||||
|
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
|
||||||
|
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
|
||||||
|
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the hub's declaration carries the peer and the takeover.
|
||||||
|
nodes, computed, err := graph(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var hubNode overlay.Node
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.Name == "anchor" {
|
||||||
|
hubNode = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
|
||||||
|
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
|
||||||
|
}
|
||||||
|
carried := false
|
||||||
|
for _, p := range computed["anchor"] {
|
||||||
|
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
|
||||||
|
carried = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !carried {
|
||||||
|
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an
|
||||||
|
// endpoint port that differs would have the host stop the found interface and raise the mesh's
|
||||||
|
// where no peer is listening.
|
||||||
|
func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := open.inventory
|
||||||
|
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hub, err := inv.NodeByName(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const key = "THE-TUNNELS-KEY========================="
|
||||||
|
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||||
|
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||||
|
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the
|
||||||
|
// tunnel over.
|
||||||
|
_, _, err = graph(ctx, open)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Re-placed on the tunnel, it composes.
|
||||||
|
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, _, err := graph(ctx, open); err != nil {
|
||||||
|
t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||||
|
// catalogue is not beside this checkout.
|
||||||
|
func theResolver(t *testing.T) catalogue.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("dnsmasq does not parse:\n%v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||||
|
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||||
|
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||||
|
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||||
|
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, theResolver(t))
|
||||||
|
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Its bus credential, as assigning issues it where the bus is reachable (novox/hq issue 203):
|
||||||
|
// no bus is known to this test, so it is minted here, or composing refuses the placeholder.
|
||||||
|
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: "anchor.dnsmasq", Kind: inventory.BusModule, Node: "anchor", Module: "dnsmasq"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
zones := func() string {
|
||||||
|
t.Helper()
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "dnsmasq.fact-node-zones" {
|
||||||
|
if r["path"] != "/etc/mesh-resolver/nodes.conf" {
|
||||||
|
t.Fatalf("the machines were written somewhere the resolver does not read: %v", r["path"])
|
||||||
|
}
|
||||||
|
return r["content"].(string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("the resolver was not handed the machines")
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
first := zones()
|
||||||
|
for _, want := range []string{
|
||||||
|
"local=/internal/", "address=/anchor.internal/10.77.0.1\n", "address=/laptop.internal/10.77.0.2\n",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(first, want) {
|
||||||
|
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||||
|
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||||
|
// container asks, read only when the runtime starts.
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "dnsmasq.runtime-dns" {
|
||||||
|
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The laptop keeps its place and its address, and stops running the network.
|
||||||
|
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after := zones()
|
||||||
|
if strings.Contains(after, "laptop") || !strings.Contains(after, "address=/anchor.internal/10.77.0.1\n") {
|
||||||
|
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||||
|
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||||
|
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||||
|
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
gens, err := generators(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"anchor", "laptop"} {
|
||||||
|
plan, settings, err := planFor(ctx, open, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||||
|
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,560 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/token"
|
||||||
|
)
|
||||||
|
|
||||||
|
// what a machine is, and what it is allowed to be told.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
func nodeCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
switch args[0] {
|
||||||
|
case "show":
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("node show <name>")
|
||||||
|
}
|
||||||
|
return showNode(ctx, inv, args[1])
|
||||||
|
case "add":
|
||||||
|
return addNode(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
case "list":
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **The same list, for something other than a person** — the console's discovery reads it
|
||||||
|
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||||
|
if len(args) > 1 && args[1] == "--json" {
|
||||||
|
type listed struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Heard string `json:"heard"`
|
||||||
|
Mode string `json:"mode"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
out := make([]listed, 0, len(nodes))
|
||||||
|
for _, n := range nodes {
|
||||||
|
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||||
|
}
|
||||||
|
return printJSON(out)
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||||
|
// look the same, and this command answering "none" is only honest because getting
|
||||||
|
// here means the store answered.
|
||||||
|
fmt.Println("this mesh has no node records yet")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, n := range nodes {
|
||||||
|
fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case "public-domain":
|
||||||
|
// The domain this node composes its routed names under (novox/hq ADR 0066).
|
||||||
|
//
|
||||||
|
// **The form with no argument reports; clearing is asked for by name.** It used to clear —
|
||||||
|
// so `node public-domain anchor`, which reads like a question and is what anybody types to
|
||||||
|
// find out what the answer is, silently took every routed name the node had. A read-shaped
|
||||||
|
// invocation must never be a destructive write: there is no output that makes up for it,
|
||||||
|
// because the damage is already done by the time it prints.
|
||||||
|
return publicDomain(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
case "networks":
|
||||||
|
// Removed by novox/hq ADR 0140, which superseded the record that added it. The filter no
|
||||||
|
// longer names any network: it constrains what arrives from outside the machine and says
|
||||||
|
// nothing about what did not, so there is no list to keep. Answered rather than met with
|
||||||
|
// "unknown command", because this was the documented way to stop a flip cutting a machine's
|
||||||
|
// containers off and somebody will reasonably still type it.
|
||||||
|
return errors.New("`node networks` is gone (novox/hq ADR 0140). The filter constrains what " +
|
||||||
|
"arrives from outside this machine and says nothing about traffic that did not, so no " +
|
||||||
|
"network is named anywhere and nothing needs to be said to keep a machine's own " +
|
||||||
|
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||||
|
"`node show <name>`")
|
||||||
|
|
||||||
|
case "account":
|
||||||
|
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||||
|
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||||
|
// an optional second argument is the home when it is not /home/<account>.
|
||||||
|
return nodeAccount(ctx, inv, args[1:])
|
||||||
|
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||||
|
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
|
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||||
|
adopted := set.Bool("adopted", false,
|
||||||
|
"the machine is in use: keep what is found on it until each module is taken")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("node add <name> [--adopted]")
|
||||||
|
}
|
||||||
|
node, err := inv.AddNodeAs(ctx, positionals[0], *adopted)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("added %s (%s)", node.Name, node.ID)
|
||||||
|
if node.Adopted {
|
||||||
|
fmt.Print(", adopted")
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted
|
||||||
|
// wherever the mesh reports a node's state.
|
||||||
|
func modeOf(n inventory.Node) string {
|
||||||
|
if n.Adopted {
|
||||||
|
return "adopted"
|
||||||
|
}
|
||||||
|
return "converged"
|
||||||
|
}
|
||||||
|
|
||||||
|
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
||||||
|
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
|
||||||
|
// argument, like public-domain: `node account novox` answers, it does not change anything.
|
||||||
|
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
|
||||||
|
if len(positionals) == 0 || len(positionals) > 3 {
|
||||||
|
return errors.New("node account <name> — what it is now; " +
|
||||||
|
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
|
||||||
|
}
|
||||||
|
node := positionals[0]
|
||||||
|
if len(positionals) == 1 {
|
||||||
|
who, err := inv.NodeByName(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if who.Account == "" {
|
||||||
|
fmt.Printf("%s has no operator account known\n", node)
|
||||||
|
fmt.Printf(" `node account %s <account>` sets it\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
home := ""
|
||||||
|
if len(positionals) == 3 {
|
||||||
|
home = positionals[2]
|
||||||
|
}
|
||||||
|
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
|
||||||
|
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||||
|
"<name> <domain> to set it; <name> --clear to take it away"
|
||||||
|
|
||||||
|
// publicDomain reads, sets or clears the domain a node composes its routed names under.
|
||||||
|
//
|
||||||
|
// Three forms, and the destructive one is the only one that has to be asked for. Clearing is a
|
||||||
|
// real thing to want — a machine that stops facing the outside composes no names, and
|
||||||
|
// lab-versus-production is this one setting (novox/hq ADR 0066) — so it keeps a way to say it.
|
||||||
|
// What it does not keep is being the thing that happens when nothing was said at all.
|
||||||
|
func publicDomain(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||||
|
set := flag.NewFlagSet("node public-domain", flag.ContinueOnError)
|
||||||
|
clear := set.Bool("clear", false, "take the domain away; it composes no routed names after")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) == 0 || len(positionals) > 2 {
|
||||||
|
return errors.New(publicDomainUsage)
|
||||||
|
}
|
||||||
|
node := positionals[0]
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case *clear && len(positionals) == 2:
|
||||||
|
// Both, which cannot be meant. Refused rather than one of them silently winning.
|
||||||
|
return fmt.Errorf("give %s a domain or --clear, not both: %q and --clear say opposite "+
|
||||||
|
"things and the mesh will not choose between them", node, positionals[1])
|
||||||
|
|
||||||
|
case *clear:
|
||||||
|
if err := inv.SetPublicDomain(ctx, node, ""); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
|
||||||
|
fmt.Printf(" run `push %s` to take them off it\n", node)
|
||||||
|
return nil
|
||||||
|
|
||||||
|
case len(positionals) == 2:
|
||||||
|
if err := inv.SetPublicDomain(ctx, node, positionals[1]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s composes its routed names under %s\n", node, positionals[1])
|
||||||
|
fmt.Printf(" run `push %s` to send it\n", node)
|
||||||
|
return nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
// Asked, so answered. NodeByName first, so a name the mesh has never heard of is a refusal
|
||||||
|
// rather than "it has no public domain", which is true of that name and says nothing.
|
||||||
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
domain, err := inv.PublicDomainOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if domain == "" {
|
||||||
|
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
|
||||||
|
fmt.Printf(" `node public-domain %s <domain>` gives it one\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s composes its routed names under %s\n", node, domain)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func tokenCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 || args[0] != "issue" {
|
||||||
|
return errors.New("token issue --node <name>, or token issue --new <name>")
|
||||||
|
}
|
||||||
|
|
||||||
|
set := flag.NewFlagSet("token issue", flag.ContinueOnError)
|
||||||
|
existing := set.String("node", "", "issue for a node record that already exists")
|
||||||
|
fresh := set.String("new", "", "create the node record, then issue for it")
|
||||||
|
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||||
|
adopted := set.Bool("adopted", false,
|
||||||
|
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||||
|
if err := set.Parse(args[1:]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exactly one, because the difference is what the token binds to. A command that guessed
|
||||||
|
// would sometimes create a second record for a machine that already has one.
|
||||||
|
if (*existing == "") == (*fresh == "") {
|
||||||
|
return errors.New("give exactly one of --node <name> or --new <name>: the first is a " +
|
||||||
|
"machine the mesh already has a record for, the second is one it has never seen")
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assembled from two contexts by the process that holds both grants. Neither reads the
|
||||||
|
// other's store (novox/hq ADR 0008) — each is asked for its own part.
|
||||||
|
ident, err := openIdentity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ident.Close()
|
||||||
|
key, err := ident.Establish(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The account is created before the token is handed over, which is what removes the
|
||||||
|
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
|
||||||
|
// exist before it does. The one-time secret IS the password, so a node's first connection is
|
||||||
|
// already authenticated and enrolment is what happens over it.
|
||||||
|
|
||||||
|
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
|
||||||
|
Adopted: issued.Node.Adopted}
|
||||||
|
|
||||||
|
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
||||||
|
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
switch {
|
||||||
|
case err == nil:
|
||||||
|
made.Broker, made.Fingerprint = known.Address, known.Fingerprint
|
||||||
|
case errors.Is(err, broker.ErrNotConfigured):
|
||||||
|
default:
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
encoded, err := made.Encode()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
joins := ""
|
||||||
|
if made.Adopted {
|
||||||
|
joins = ", joining adopted"
|
||||||
|
}
|
||||||
|
fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n",
|
||||||
|
issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded)
|
||||||
|
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
||||||
|
|
||||||
|
if missing := made.Missing(); len(missing) > 0 {
|
||||||
|
fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n")
|
||||||
|
for _, m := range missing {
|
||||||
|
fmt.Printf(" - %s\n", m)
|
||||||
|
}
|
||||||
|
fmt.Printf("\nSet %s and %s once the broker is raised.\n",
|
||||||
|
broker.AddressVar, broker.CertificateVar)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||||
|
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||||
|
// its mode, which is what the token says.
|
||||||
|
func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool,
|
||||||
|
validFor time.Duration) (inventory.Issued, error) {
|
||||||
|
name := existing
|
||||||
|
if fresh != "" {
|
||||||
|
node, err := inv.AddNodeAs(ctx, fresh, adopted)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Issued{}, err
|
||||||
|
}
|
||||||
|
name = node.Name
|
||||||
|
}
|
||||||
|
// Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not
|
||||||
|
// converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a
|
||||||
|
// node already converged is refused rather than done quietly: returning a node to adopted is
|
||||||
|
// its own act too, which unloads the mesh's filter and enables the found firewall again.
|
||||||
|
if adopted && fresh == "" {
|
||||||
|
node, err := inv.NodeByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Issued{}, err
|
||||||
|
}
|
||||||
|
if !node.Adopted {
|
||||||
|
return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+
|
||||||
|
"that: run `adopt %s` to return it to adopted, then issue the token without "+
|
||||||
|
"--adopted", name, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return inv.IssueToken(ctx, name, validFor)
|
||||||
|
}
|
||||||
|
|
||||||
|
func identityCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 || args[0] != "show" {
|
||||||
|
return errors.New("identity show")
|
||||||
|
}
|
||||||
|
ident, err := openIdentity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer ident.Close()
|
||||||
|
|
||||||
|
// Establish rather than read: a control plane asked for its identity before it has one should
|
||||||
|
// get one, not an error. Generating it is idempotent, so this is safe to run at any time.
|
||||||
|
key, err := ident.Establish(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("signing key %s\n", key.ID)
|
||||||
|
fmt.Printf("fingerprint %s\n", key.Fingerprint())
|
||||||
|
fmt.Printf("created %s\n", key.Created.Format(time.RFC3339))
|
||||||
|
fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" +
|
||||||
|
"declaration because it carries a signature this key made (novox/hq ADR 0004).\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func brokerCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) > 0 && args[0] == "certificate" {
|
||||||
|
return busCertificate(args[1:])
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "accounts" {
|
||||||
|
return busAccounts(ctx, args[1:])
|
||||||
|
}
|
||||||
|
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||||
|
return consumerReset(args[1:])
|
||||||
|
}
|
||||||
|
if len(args) == 0 || args[0] != "show" {
|
||||||
|
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||||
|
"broker consumer-reset <stream> <consumer>")
|
||||||
|
}
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
if errors.Is(err, broker.ErrNotConfigured) {
|
||||||
|
fmt.Printf("no broker configured. Set %s and %s.\n\n"+
|
||||||
|
"Until then tokens carry the signing key and the one-time secret, and say what they\n"+
|
||||||
|
"are missing. They cannot be used to join.\n",
|
||||||
|
broker.AddressVar, broker.CertificateVar)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("address %s\n", known.Address)
|
||||||
|
fmt.Printf("fingerprint %s\n", known.Fingerprint)
|
||||||
|
fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" +
|
||||||
|
"node checks it before sending anything (novox/hq ADR 0004).\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||||
|
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||||
|
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||||
|
func consumerReset(args []string) error {
|
||||||
|
if len(args) != 2 {
|
||||||
|
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||||
|
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||||
|
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||||
|
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||||
|
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||||
|
//
|
||||||
|
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||||
|
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
||||||
|
// picture of the mesh.
|
||||||
|
func heardFrom(n inventory.Node) string {
|
||||||
|
silent, ever := n.Silent()
|
||||||
|
switch {
|
||||||
|
case !ever:
|
||||||
|
return "never spoken"
|
||||||
|
case silent > SilentFor:
|
||||||
|
return "out of touch " + roughly(silent)
|
||||||
|
default:
|
||||||
|
return "here"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// roughly is a duration a person reads rather than parses.
|
||||||
|
func roughly(d time.Duration) string {
|
||||||
|
switch {
|
||||||
|
case d < time.Hour:
|
||||||
|
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||||
|
case d < 48*time.Hour:
|
||||||
|
return fmt.Sprintf("%dh", int(d.Hours()))
|
||||||
|
default:
|
||||||
|
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// showNode says what one machine reported about itself, in its own words.
|
||||||
|
//
|
||||||
|
// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what
|
||||||
|
// a machine can do is the one it gave — and its detail is half of that account. The mesh was
|
||||||
|
// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with
|
||||||
|
// nowhere to go: a person told a machine lacks something wants to know what the detector saw.
|
||||||
|
//
|
||||||
|
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||||
|
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||||
|
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||||
|
node, err := inv.NodeByName(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s\n", node.Name)
|
||||||
|
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
||||||
|
// Which host runs it, as it reported (novox/hq 04-ISSUES/087). Said whenever known, because a
|
||||||
|
// host refuses a declaration carrying a field it does not understand and refuses it WHOLE — so
|
||||||
|
// which host a machine runs is what decides whether the mesh can send it anything new, and
|
||||||
|
// nothing could say it. "not reported" rather than blank: a machine that has not said is a
|
||||||
|
// different thing from one running nothing.
|
||||||
|
fmt.Printf(" host %s\n", orNotReported(node.HostVersion))
|
||||||
|
if err := showMode(ctx, inv, node); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||||
|
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||||
|
// every internal node would be noise.
|
||||||
|
domain, err := inv.PublicDomainOf(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if domain != "" {
|
||||||
|
fmt.Printf(" public domain %s\n", domain)
|
||||||
|
}
|
||||||
|
|
||||||
|
held, err := inv.Profile(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if held == nil {
|
||||||
|
// Never reported is not the same as reported nothing, and the remedy differs: one is a
|
||||||
|
// machine that has not run the host yet, the other is a machine that ran it and can do
|
||||||
|
// nothing.
|
||||||
|
fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" +
|
||||||
|
" capability is refused here — run the host on it\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(held) == 0 {
|
||||||
|
fmt.Printf("\n it reported no capabilities at all\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("\n what it can do, as it reported:\n")
|
||||||
|
for _, c := range held {
|
||||||
|
mark := "no "
|
||||||
|
if c.Present {
|
||||||
|
mark = "yes"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail)
|
||||||
|
}
|
||||||
|
|
||||||
|
assigned, err := inv.Assigned(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(assigned) > 0 {
|
||||||
|
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// orNotReported is a fact a machine states about itself, or the fact that it has not.
|
||||||
|
func orNotReported(s string) string {
|
||||||
|
if strings.TrimSpace(s) == "" {
|
||||||
|
return "not reported — this machine has not said since the mesh began keeping it"
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||||
|
func printJSON(v any) error {
|
||||||
|
body, err := json.MarshalIndent(v, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **A read-shaped invocation is never a destructive write.**
|
||||||
|
//
|
||||||
|
// `node public-domain anchor` used to clear the domain. It reads like a question — it is what
|
||||||
|
// anybody types to find out what the answer is — and it silently took every routed name the node
|
||||||
|
// had. There is no output that makes up for that: by the time it prints, the fact is gone.
|
||||||
|
func TestAskingForANodesPublicDomainDoesNotTakeItAway(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := publicDomain(ctx, open.inventory, []string{"anchor"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if domain != "example.test" {
|
||||||
|
t.Fatalf("asking what the domain is took it away: %q", domain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clearing is a real thing to want — a machine that stops facing the outside composes no names —
|
||||||
|
// so it keeps a way to be said. What it stops being is what happens when nothing was said.
|
||||||
|
func TestClearingANodesPublicDomainIsAskedForByName(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := publicDomain(ctx, open.inventory, []string{"anchor", "--clear"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if domain != "" {
|
||||||
|
t.Fatalf("--clear did not clear it: %q", domain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A domain sets it, as it always did.
|
||||||
|
func TestGivingANodeAPublicDomainSetsIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := publicDomain(ctx, open.inventory, []string{"anchor", "example.test"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if domain != "example.test" {
|
||||||
|
t.Fatalf("got %q", domain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A domain and --clear say opposite things. Refused rather than one of them silently winning.
|
||||||
|
func TestADomainAndClearTogetherIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := publicDomain(ctx, open.inventory, []string{"anchor", "other.test", "--clear"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a domain and --clear together were accepted")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "not both") {
|
||||||
|
t.Fatalf("the refusal does not say why: %v", err)
|
||||||
|
}
|
||||||
|
// And neither half happened.
|
||||||
|
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if domain != "example.test" {
|
||||||
|
t.Fatalf("a refused command changed something: %q", domain)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Asking about a name the mesh has never heard of is a refusal, not "it has no public domain" —
|
||||||
|
// which is true of that name and says nothing.
|
||||||
|
func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
if err := publicDomain(t.Context(), open.inventory, []string{"nowhere"}); err == nil {
|
||||||
|
t.Fatal("a name the mesh does not know was answered as if it were a machine")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: the operator says a node is adopted — `node add --adopted` for a record, and
|
||||||
|
// the token for a machine joining.
|
||||||
|
func TestANodeAddedAdoptedIsAdopted(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
if err := addNode(ctx, open.inventory, []string{"joiner", "--adopted"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
n, err := open.inventory.NodeByName(ctx, "joiner")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !n.Adopted {
|
||||||
|
t.Fatal("node add --adopted made a converged node")
|
||||||
|
}
|
||||||
|
if err := addNode(ctx, open.inventory, []string{"plain"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "plain"); n.Adopted {
|
||||||
|
t.Fatal("node add without --adopted made an adopted node")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
issued, err := issueFor(ctx, open.inventory, "", "joiner", true, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !issued.Node.Adopted {
|
||||||
|
t.Fatal("a token issued --adopted is for a node that is not adopted")
|
||||||
|
}
|
||||||
|
again, err := issueFor(ctx, open.inventory, "joiner", "", false, time.Hour)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !again.Node.Adopted {
|
||||||
|
t.Fatal("re-issuing without --adopted converged the node; converging is its own act")
|
||||||
|
}
|
||||||
|
// --adopted for a node already converged is refused, and points at the act that does it.
|
||||||
|
if _, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "adopt laptop") {
|
||||||
|
t.Fatalf("--adopted on a converged node was not refused: %v", err)
|
||||||
|
}
|
||||||
|
if n, _ := open.inventory.NodeByName(ctx, "laptop"); n.Adopted {
|
||||||
|
t.Fatal("a refused token flipped the node to adopted")
|
||||||
|
}
|
||||||
|
// And said for a node that is adopted already, it is the ordinary re-issue.
|
||||||
|
if _, err := issueFor(ctx, open.inventory, "joiner", "", true, time.Hour); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,307 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// operatorCommand is the mesh's one holder of secrets that is not a machine.
|
||||||
|
//
|
||||||
|
// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
|
||||||
|
// is gone takes its secrets with it** — the store's superuser and the broker's administrator among
|
||||||
|
// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key
|
||||||
|
// whose private half is made where the operator is and never enters the mesh. Making the key and
|
||||||
|
// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs
|
||||||
|
// wherever the operator keeps things; the second gives the mesh the public half and nothing else.
|
||||||
|
// The controller's own container is a scratch image with no writable path, which is the right
|
||||||
|
// shape for a program that must hold no key — so the private half could not be written there
|
||||||
|
// even by mistake.
|
||||||
|
//
|
||||||
|
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
|
||||||
|
// operator key set <public> tell the mesh which key to seal to
|
||||||
|
// operator key show the public key, its fingerprint, and what it can recover
|
||||||
|
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | " +
|
||||||
|
"operator key show | operator issue <name> --invokes <tool,tool|*> | operator revoke <name> | " +
|
||||||
|
"operator list"
|
||||||
|
|
||||||
|
func operatorCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New(operatorUsage)
|
||||||
|
}
|
||||||
|
// The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because
|
||||||
|
// both answer "who, other than a machine, may do something here" — and a person reading this
|
||||||
|
// command's usage is asking exactly that.
|
||||||
|
switch args[0] {
|
||||||
|
case "issue":
|
||||||
|
return personIssue(ctx, args[1:])
|
||||||
|
case "revoke":
|
||||||
|
return personRevoke(ctx, args[1:])
|
||||||
|
case "list":
|
||||||
|
return personList(ctx)
|
||||||
|
}
|
||||||
|
if len(args) < 2 || args[0] != "key" {
|
||||||
|
return errors.New(operatorUsage)
|
||||||
|
}
|
||||||
|
switch args[1] {
|
||||||
|
case "make":
|
||||||
|
return operatorKeyMake(args[2:])
|
||||||
|
case "set":
|
||||||
|
return operatorKeySet(ctx, args[2:])
|
||||||
|
case "show":
|
||||||
|
return operatorKeyShow(ctx)
|
||||||
|
default:
|
||||||
|
return errors.New(operatorUsage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live.
|
||||||
|
func operatorKeyMake(args []string) error {
|
||||||
|
set := flag.NewFlagSet("operator key make", flag.ContinueOnError)
|
||||||
|
out := set.String("out", "operator.key",
|
||||||
|
"where to write the private key (0600); keep it off the mesh, and keep it")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
public, private, err := secrets.Keypair()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Create-exclusive: a key somebody may still need is never overwritten, and there is no window
|
||||||
|
// between checking and writing in which one could appear.
|
||||||
|
if err := writeNew(*out, []byte(private+"\n")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
|
||||||
|
fmt.Printf(" private half written to %s (0600) — keep it off the mesh, and keep it\n", *out)
|
||||||
|
fmt.Printf(" public half, to give the mesh with `operator key set`:\n")
|
||||||
|
fmt.Printf("public %s\n", public)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func operatorKeySet(ctx context.Context, args []string) error {
|
||||||
|
rest, flags := split(args)
|
||||||
|
set := flag.NewFlagSet("operator key set", flag.ContinueOnError)
|
||||||
|
replace := set.Bool("replace", false,
|
||||||
|
"replace an existing operator key — secrets sealed to the old one stay sealed to it")
|
||||||
|
if err := set.Parse(flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 1 {
|
||||||
|
return errors.New(operatorUsage)
|
||||||
|
}
|
||||||
|
public := strings.TrimSpace(rest[0])
|
||||||
|
if _, err := secrets.Seal(public, []byte("probe")); err != nil {
|
||||||
|
return fmt.Errorf("that is not a public sealing key: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
if current, err := inv.OperatorKey(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
} else if current != "" && current != public && !*replace {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"the mesh already has an operator key (%s). Pass --replace to change it — "+
|
||||||
|
"secrets sealed to the current key stay sealed to it until each is issued again",
|
||||||
|
secrets.Fingerprint(current))
|
||||||
|
}
|
||||||
|
orphaned, err := inv.SetOperatorKey(ctx, public)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
|
||||||
|
fmt.Printf(" the mesh holds the public half only and cannot open what it seals to it;\n")
|
||||||
|
fmt.Printf(" from now on every secret a module holds for itself is sealed to it as well.\n")
|
||||||
|
fmt.Printf(" Secrets made before this cannot be — each is recoverable once issued again\n")
|
||||||
|
if orphaned > 0 {
|
||||||
|
fmt.Printf(" %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func operatorKeyShow(ctx context.Context) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
key, err := inv.OperatorKey(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if key == "" {
|
||||||
|
fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
kept, earlier, unrecoverable, err := inv.KeptForOperator(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("operator key %s\n %s\n", secrets.Fingerprint(key), key)
|
||||||
|
fmt.Printf(" %d secret(s) recoverable with it\n", len(kept))
|
||||||
|
if len(earlier) > 0 {
|
||||||
|
fmt.Printf(" %d secret(s) sealed to an earlier operator key — recoverable with that key only, until issued again:\n", len(earlier))
|
||||||
|
for _, k := range earlier {
|
||||||
|
fmt.Printf(" %s %s %s (%s)\n", k.Node, k.Module, k.Name, secrets.Fingerprint(k.Key))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(unrecoverable) > 0 {
|
||||||
|
fmt.Printf(" %d secret(s) not recoverable — made before the mesh had an operator key:\n", len(unrecoverable))
|
||||||
|
for _, k := range unrecoverable {
|
||||||
|
fmt.Printf(" %s %s %s\n", k.Node, k.Module, k.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readPrivateKey is the operator's key from the file `operator key make` wrote.
|
||||||
|
func readPrivateKey(path string) (string, error) {
|
||||||
|
if path == "" {
|
||||||
|
return "", errors.New("--key <file> names the operator's private key, written by `operator key make`")
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(raw)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// personIssue gives somebody a credential for the mesh's tools, and prints it once.
|
||||||
|
//
|
||||||
|
// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the
|
||||||
|
// credential exists anywhere but on the workstation that will use it — the same contract a token has,
|
||||||
|
// and for the same reason: a credential recoverable from the mesh's store has the store's blast
|
||||||
|
// radius.
|
||||||
|
func personIssue(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
|
||||||
|
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
|
||||||
|
// Flags on either side of the name, because the usage this command prints puts them after it —
|
||||||
|
// and the standard parser stops at the first thing that is not a flag, so the order the command
|
||||||
|
// documents was the one order it refused (2026-09-28).
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("operator issue <name> --invokes <tool,tool|*>")
|
||||||
|
}
|
||||||
|
name := positionals[0]
|
||||||
|
if *invokes == "" {
|
||||||
|
return errors.New(
|
||||||
|
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
|
||||||
|
"or --invokes '*' for an administrator")
|
||||||
|
}
|
||||||
|
var tools []string
|
||||||
|
for _, t := range strings.Split(*invokes, ",") {
|
||||||
|
if t = strings.TrimSpace(t); t != "" {
|
||||||
|
tools = append(tools, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Refused here rather than at the next composition, where it would stop the whole file being
|
||||||
|
// written for everybody. A name that cannot be part of a subject is one the server would read as
|
||||||
|
// a wider permission than anybody granted.
|
||||||
|
if _, err := broker.PermissionsFor(broker.Principal{
|
||||||
|
Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x",
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username()
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
where, err := broker.FromEnvironment()
|
||||||
|
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
held, err := json.Marshal(struct {
|
||||||
|
URL string `json:"url"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
User string `json:"user"`
|
||||||
|
Password string `json:"password"`
|
||||||
|
Person string `json:"person"`
|
||||||
|
Invokes []string `json:"invokes"`
|
||||||
|
}{
|
||||||
|
URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||||
|
User: user, Password: password, Person: name, Invokes: tools,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", "))
|
||||||
|
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||||
|
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println(string(held))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func personRevoke(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("operator revoke <name>")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops
|
||||||
|
// working when the file no longer names it. Said plainly, because "revoked" that still works for
|
||||||
|
// another minute is worth knowing about.
|
||||||
|
fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+
|
||||||
|
"push the machine holding mesh-broker to make it so\n", args[0])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func personList(ctx context.Context) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
people, err := open.inventory.People(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(people) == 0 {
|
||||||
|
fmt.Println("nobody but machines reaches this mesh")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, p := range people {
|
||||||
|
fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", "))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// entry is a module as the catalogue holds it: built, so its manifest carries no `build` any more.
|
||||||
|
func entry(module string, _ ...string) inventory.Entry {
|
||||||
|
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stoodOn is what each module's newest build recorded it was handed.
|
||||||
|
func stoodOn(edges map[string][]string) map[string][]string {
|
||||||
|
out := map[string][]string{}
|
||||||
|
for module, bases := range edges {
|
||||||
|
for _, b := range bases {
|
||||||
|
out[module] = append(out[module], catalogue.ArtifactStoreScheme+b+"/runtime@sha256:"+strings.Repeat("0", 64))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module built before the module it stands on is built against the old one and reports success
|
||||||
|
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
|
||||||
|
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||||
|
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
|
||||||
|
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
|
||||||
|
got := orderByBases(in, edges)
|
||||||
|
pos := map[string]int{}
|
||||||
|
for i, e := range got {
|
||||||
|
pos[e.Manifest.Module] = i
|
||||||
|
}
|
||||||
|
if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) {
|
||||||
|
t.Fatalf("bases not first: %v", pos)
|
||||||
|
}
|
||||||
|
if len(got) != 4 {
|
||||||
|
t.Fatalf("an entry was lost or doubled: %d", len(got))
|
||||||
|
}
|
||||||
|
// A base outside the set is not waited for: it is not being rebuilt.
|
||||||
|
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("a dependency outside the set changed the set: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module registered from its manifest and never built still names its bases there; once built,
|
||||||
|
// the recorded edge is what says so. Both are read, and a module never stands on itself.
|
||||||
|
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
|
||||||
|
built := entry("gitea")
|
||||||
|
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
|
||||||
|
if !standsOnModule(built, "mesh-tools", edges) {
|
||||||
|
t.Fatal("a recorded edge was not read")
|
||||||
|
}
|
||||||
|
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
|
||||||
|
t.Fatal("an edge was invented")
|
||||||
|
}
|
||||||
|
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
|
||||||
|
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||||
|
}}}
|
||||||
|
if !standsOnModule(fresh, "mesh-tools", nil) {
|
||||||
|
t.Fatal("a manifest's own base was not read")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge names a repository the way the forge does; a source is recorded the way a build was
|
||||||
|
// asked for. The two meet on owner/repo and branch, whichever form the record took.
|
||||||
|
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||||
|
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||||
|
for _, s := range []inventory.Source{
|
||||||
|
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"},
|
||||||
|
{Repository: "novox/mesh-controller", Seat: "git", Ref: ""},
|
||||||
|
{Repository: "https://elsewhere.example/novox/mesh-controller", Ref: "main"},
|
||||||
|
} {
|
||||||
|
if !sourceIs(s, m) {
|
||||||
|
t.Errorf("%+v was not matched by the merge", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, s := range []inventory.Source{
|
||||||
|
{Repository: "novox/mesh-host", Seat: "git"},
|
||||||
|
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "release"},
|
||||||
|
} {
|
||||||
|
if sourceIs(s, m) {
|
||||||
|
t.Errorf("%+v was matched by a merge that is not its", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge made before the source was last seen is history: it does not move the source, and a
|
||||||
|
// merge that says nothing about when it was made is taken as news.
|
||||||
|
func TestAMergeOlderThanTheLastLookIsHistory(t *testing.T) {
|
||||||
|
seen := time.Date(2026, 9, 28, 3, 0, 0, 0, time.UTC)
|
||||||
|
if !isHistory("2026-09-28T02:00:00Z", seen) {
|
||||||
|
t.Fatal("an older merge was taken as news")
|
||||||
|
}
|
||||||
|
if isHistory("2026-09-28T04:00:00Z", seen) {
|
||||||
|
t.Fatal("a newer merge was taken as history")
|
||||||
|
}
|
||||||
|
if isHistory("", seen) || isHistory("2026-09-28T02:00:00Z", time.Time{}) {
|
||||||
|
t.Fatal("a merge or a source with no time on it was refused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module as the catalogue holds it: built from a repository, at a directory inside it.
|
||||||
|
func fromRepo(module, repository, path string) inventory.Entry {
|
||||||
|
return inventory.Entry{
|
||||||
|
Manifest: catalogue.Manifest{Module: module},
|
||||||
|
Source: inventory.Source{Repository: repository, Path: path, Ref: "main"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A merge rebuilds the modules whose own directories it changed, and everything when what it changed
|
||||||
|
// is shared. One repository holding many modules is the ordinary case here, and rebuilding all of
|
||||||
|
// them for a change to one is what exhausted a registry's pull limit the first night this ran.
|
||||||
|
func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||||
|
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||||
|
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||||
|
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||||
|
known := []inventory.Entry{gitea, keycloak, fromRepo("plex", repo, "modules/plex")}
|
||||||
|
candidates := []inventory.Entry{gitea, keycloak}
|
||||||
|
merge := func(paths []string, truncated bool) link.SourceMoved {
|
||||||
|
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main",
|
||||||
|
Paths: paths, PathsTruncated: truncated}
|
||||||
|
}
|
||||||
|
named := func(entries []inventory.Entry) string {
|
||||||
|
var names []string
|
||||||
|
for _, e := range entries {
|
||||||
|
names = append(names, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
return strings.Join(names, ",")
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
m link.SourceMoved
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
|
||||||
|
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
|
||||||
|
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||||
|
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||||
|
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||||
|
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||||
|
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||||
|
// shows it is one — and a directory that may be shared code is still shared.
|
||||||
|
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||||
|
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||||
|
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||||
|
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
|
||||||
|
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
|
||||||
|
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||||
|
} {
|
||||||
|
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||||
|
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module whose recipe packages source from another repository is affected when that repository
|
||||||
|
// moves — the manifest the mesh keeps says nothing about it, so the record of what the build read is
|
||||||
|
// the only thing that can say so.
|
||||||
|
func TestAModuleIsAffectedByTheRepositoryItPackages(t *testing.T) {
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||||
|
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||||
|
for _, read := range [][]inventory.ReadRepository{
|
||||||
|
{{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}},
|
||||||
|
{{Repository: "novox/mesh-controller"}},
|
||||||
|
{{Repository: "https://elsewhere.example/novox/other"}, {Repository: "novox/mesh-controller.git", Ref: "main"}},
|
||||||
|
} {
|
||||||
|
if !readsFrom(read, m) {
|
||||||
|
t.Errorf("%+v was not matched by the merge", read)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, read := range [][]inventory.ReadRepository{
|
||||||
|
nil,
|
||||||
|
{{Repository: "novox/mesh-host", Ref: "main"}},
|
||||||
|
{{Repository: "novox/mesh-controller", Ref: "release"}},
|
||||||
|
} {
|
||||||
|
if readsFrom(read, m) {
|
||||||
|
t.Errorf("%+v was matched by a merge that is not its", read)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What a module handed over by hand records about where it came from, and what is refused.
|
||||||
|
func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||||
|
// The whole location: a repository on the mesh's own forge, the directory inside it, the branch
|
||||||
|
// and the commit the manifest was read at.
|
||||||
|
from, err := whereItComesFrom("novox/mesh-catalog", "main", "c0ffee", "modules/gitea", true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if from.Path != "modules/gitea" || from.Seat != "git" || from.Repository != "novox/mesh-catalog" {
|
||||||
|
t.Fatalf("the source records as %+v", from)
|
||||||
|
}
|
||||||
|
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
|
||||||
|
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
|
||||||
|
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
what string
|
||||||
|
repository, ref, commit, path string
|
||||||
|
self bool
|
||||||
|
}{
|
||||||
|
{what: "a source with no commit", repository: "novox/mesh-catalog", commit: ""},
|
||||||
|
{what: "a commit with no source", commit: "c0ffee"},
|
||||||
|
{what: "a directory inside nothing", path: "modules/gitea"},
|
||||||
|
{what: "a forge holding nothing", self: true},
|
||||||
|
{what: "an address given as a path on the forge", repository: "http://forge.internal:20000/novox/x.git", commit: "c0ffee", self: true},
|
||||||
|
} {
|
||||||
|
if _, err := whereItComesFrom(c.repository, c.ref, c.commit, c.path, c.self); err == nil {
|
||||||
|
t.Errorf("%s was recorded as a source", c.what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/215: a module once built at a commit still follows its branch — a merge into it
|
||||||
|
// matches the module, and a plan re-asks the branch, not the old commit.
|
||||||
|
func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||||
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main"}
|
||||||
|
pinned := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a"}
|
||||||
|
if !sourceIs(pinned, m) {
|
||||||
|
t.Error("a module whose record names a commit is left out of a merge into its branch")
|
||||||
|
}
|
||||||
|
full := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a1d2c3b4a5f60718293a4b5c6d7e8f9012"}
|
||||||
|
if !sourceIs(full, m) {
|
||||||
|
t.Error("a full commit hash is read as a branch")
|
||||||
|
}
|
||||||
|
if got := followedBranch("9c97a8a"); got != "" {
|
||||||
|
t.Errorf("a plan would re-ask the old commit %q", got)
|
||||||
|
}
|
||||||
|
if got := followedBranch("release"); got != "release" {
|
||||||
|
t.Errorf("a branch is not followed as named: %q", got)
|
||||||
|
}
|
||||||
|
// A module that follows another branch is still not this merge's.
|
||||||
|
if sourceIs(inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "release"}, m) {
|
||||||
|
t.Error("a module following another branch was matched")
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,401 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||||
|
//
|
||||||
|
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||||
|
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||||
|
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||||
|
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||||
|
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||||
|
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||||
|
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||||
|
// rebuild has already replaced.
|
||||||
|
|
||||||
|
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||||
|
// whether that is every holder.
|
||||||
|
type pauseView struct {
|
||||||
|
Nodes []string
|
||||||
|
All bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||||
|
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||||
|
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||||
|
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
var asked *time.Time
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||||
|
if asked == nil || s.AskedAt.Before(*asked) {
|
||||||
|
asked = s.AskedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if asked == nil {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
waited := now.Sub(*asked)
|
||||||
|
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||||
|
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||||
|
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||||
|
// longer than a day is named.
|
||||||
|
if waited > pausedTooLong {
|
||||||
|
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||||
|
}
|
||||||
|
return said, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||||
|
const pausedTooLong = 24 * time.Hour
|
||||||
|
|
||||||
|
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||||
|
// where the seat being paused changes what a plan says.
|
||||||
|
func awaitsABuild(plans []inventory.Plan) bool {
|
||||||
|
for _, p := range plans {
|
||||||
|
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||||
|
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||||
|
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||||
|
// reads as late, which is what it said before this existed.
|
||||||
|
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||||
|
if !awaitsABuild(plans) {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
seat := buildSeatAmong(entries)
|
||||||
|
holders := holdersAmong(entries, seat)
|
||||||
|
if len(holders) == 0 {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
said, err := link.PausedSaid(js, seat, holders)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||||
|
return pauseView{}
|
||||||
|
}
|
||||||
|
return pauseOf(holders, said)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pauseOf is the view from what each holder said.
|
||||||
|
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||||
|
var v pauseView
|
||||||
|
for _, n := range holders {
|
||||||
|
if said[n].Paused {
|
||||||
|
v.Nodes = append(v.Nodes, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(v.Nodes)
|
||||||
|
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||||
|
func failedIn(p inventory.Plan) []string {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||||
|
// that holds what this one held now (issue 254, ADR 0218).
|
||||||
|
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||||
|
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return q, true
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||||
|
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||||
|
switch {
|
||||||
|
case p.State == inventory.PlanDone:
|
||||||
|
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||||
|
case p.State == inventory.PlanSuperseded:
|
||||||
|
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||||
|
case p.Open():
|
||||||
|
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||||
|
}
|
||||||
|
if len(failedIn(p)) > 0 {
|
||||||
|
if q, found := newerOpenPlan(p, plans); found {
|
||||||
|
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||||
|
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
stopped := stoppedRollouts(p)
|
||||||
|
if len(stopped) == 0 {
|
||||||
|
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||||
|
p.Tier, p.ID, p.Note)
|
||||||
|
}
|
||||||
|
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||||
|
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||||
|
for _, m := range stopped {
|
||||||
|
if q, found := newerPlanFor(m, p, plans); found {
|
||||||
|
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||||
|
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||||
|
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||||
|
func stoppedRollouts(p inventory.Plan) []string {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||||
|
len(s.First) > 0 && s.Why != "" {
|
||||||
|
out = append(out, m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||||
|
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
for _, q := range plans {
|
||||||
|
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, tier := range q.Tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
if m == module {
|
||||||
|
return q, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||||
|
// variable so a test of a retried rollout needs no machine.
|
||||||
|
var sendRollout = sendToEach
|
||||||
|
|
||||||
|
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||||
|
func resumed(p *inventory.Plan, why string) {
|
||||||
|
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||||
|
p.State = inventory.PlanBuilding
|
||||||
|
p.Note = why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||||
|
// that tier: what follows is the plan going on as if they had built the first time.
|
||||||
|
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
p, err := inv.PlanByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
recent, err := inv.RecentPlans(ctx, 50)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
plans = append(plans, recent...)
|
||||||
|
if err := retryRefusal(p, plans); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if len(failedIn(p)) == 0 {
|
||||||
|
return retryRollouts(ctx, open, &p)
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
failed := failedIn(p)
|
||||||
|
var asked []string
|
||||||
|
for _, m := range failed {
|
||||||
|
askModule(ctx, &p, m, byName)
|
||||||
|
if s := p.Modules[m]; s.State == "asked" {
|
||||||
|
asked = append(asked, m+" as "+s.Build)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanBuilding {
|
||||||
|
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||||
|
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||||
|
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||||
|
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
openPlans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
recent, err := inv.RecentPlans(ctx, 20)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
p, found := planHolding(module, openPlans, recent)
|
||||||
|
if !found {
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
byName := map[string]inventory.Entry{}
|
||||||
|
for _, e := range entries {
|
||||||
|
byName[e.Manifest.Module] = e
|
||||||
|
}
|
||||||
|
was := p.State
|
||||||
|
askModule(ctx, &p, module, byName)
|
||||||
|
s := p.Modules[module]
|
||||||
|
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||||
|
if err := inv.SavePlan(ctx, p); err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
if s.State != "asked" {
|
||||||
|
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||||
|
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||||
|
switch {
|
||||||
|
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||||
|
said += "; the plan had failed and builds again from this tier"
|
||||||
|
case was == inventory.PlanFailed:
|
||||||
|
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||||
|
}
|
||||||
|
return true, said, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||||
|
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||||
|
// repository supersedes.
|
||||||
|
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||||
|
holds := func(p inventory.Plan) bool {
|
||||||
|
if p.Tier >= len(p.Tiers) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, m := range p.Tiers[p.Tier] {
|
||||||
|
if m == module {
|
||||||
|
s := p.Modules[m]
|
||||||
|
return s == nil || s.State != "built"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range openPlans {
|
||||||
|
if holds(p) {
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sorted := append([]inventory.Plan(nil), recent...)
|
||||||
|
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||||
|
for _, p := range sorted {
|
||||||
|
if p.State != inventory.PlanFailed || !holds(p) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return p, true
|
||||||
|
}
|
||||||
|
return inventory.Plan{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||||
|
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||||
|
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||||
|
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||||
|
var said []string
|
||||||
|
for _, m := range stoppedRollouts(*p) {
|
||||||
|
s := p.Modules[m]
|
||||||
|
sent, err := sendRollout(ctx, open, s.First)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||||
|
m, strings.Join(s.First, ", "), p.ID, err)
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
s.First, s.FirstAt, s.Why = sent, &now, ""
|
||||||
|
said = append(said, m+" to "+strings.Join(sent, ", "))
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanRolling
|
||||||
|
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||||
|
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||||
|
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// `plan` tells a person what a module would open and why, from the same `why` every listens
|
||||||
|
// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a
|
||||||
|
// module does not need reading its manifest first.
|
||||||
|
func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) {
|
||||||
|
m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{
|
||||||
|
{Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"},
|
||||||
|
{Port: 9001, From: catalogue.FromMesh},
|
||||||
|
}}
|
||||||
|
got := listensLines(m)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("two listens entries, got %d: %v", len(got), got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") {
|
||||||
|
t.Errorf("the port and its why did not both appear: %q", got[0])
|
||||||
|
}
|
||||||
|
if strings.Contains(got[1], "—") {
|
||||||
|
t.Errorf("a listens entry with no why should not print a dash: %q", got[1])
|
||||||
|
}
|
||||||
|
if !strings.Contains(got[1], "9001/tcp") {
|
||||||
|
t.Errorf("the port still appears without a why: %q", got[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
|
||||||
|
if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 {
|
||||||
|
t.Errorf("a module with no listens should print nothing, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
|
||||||
|
// mesh's region of a hosts file reads as the whole file.
|
||||||
|
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
|
||||||
|
region := shownAs(map[string]any{
|
||||||
|
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
|
||||||
|
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
|
||||||
|
t.Errorf("the region is shown as %q", region)
|
||||||
|
}
|
||||||
|
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
|
||||||
|
if strings.Contains(whole, "written into") {
|
||||||
|
t.Errorf("a whole file is shown as written into: %q", whole)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
|
||||||
|
// process both run the plan timer on one store. Only the one holding the plans moves them; the other
|
||||||
|
// leaves them alone, and moves them once they are let go.
|
||||||
|
func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
now := time.Now().UTC()
|
||||||
|
// Every tier done: the next step is the plan's last, and needs nothing but the store.
|
||||||
|
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||||
|
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The other controller: its own connections to the same store, holding the plans.
|
||||||
|
other, err := inventory.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(other.Close)
|
||||||
|
release, err := other.HoldPlans(ctx, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(release) // before the close above: a pool waits for a connection still held
|
||||||
|
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
|
||||||
|
t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
release()
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
|
||||||
|
t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// One machine that cannot be worked out is not a reason to leave the mesh unconverged.
|
||||||
|
//
|
||||||
|
// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on
|
||||||
|
// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine
|
||||||
|
// from being sent anything — machines with no relation to the fault, and nothing wrong with them.
|
||||||
|
// The failure and the punishment were on different machines.
|
||||||
|
//
|
||||||
|
// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on
|
||||||
|
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||||
|
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||||
|
sending, refusals := composeEach(
|
||||||
|
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||||
|
func(node string) (sendable, error) {
|
||||||
|
if node == "anchor" {
|
||||||
|
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||||
|
}
|
||||||
|
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||||
|
})
|
||||||
|
|
||||||
|
var told []string
|
||||||
|
for _, s := range sending {
|
||||||
|
told = append(told, s.node)
|
||||||
|
}
|
||||||
|
if strings.Join(told, ",") != "home-server,laptop" {
|
||||||
|
t.Errorf("a machine with nothing wrong with it was not sent: %v", told)
|
||||||
|
}
|
||||||
|
if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") ||
|
||||||
|
!strings.Contains(refusals[0], "acme-ca") {
|
||||||
|
t.Errorf("the machine that could not be worked out was not named with its reason: %v",
|
||||||
|
refusals)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||||
|
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||||
|
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||||
|
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||||
|
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||||
|
func(string) (sendable, error) { return sendable{}, nil })
|
||||||
|
if len(sending) != 1 || len(refusals) != 0 {
|
||||||
|
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A push that skipped a machine still ends badly, and says what was sent.
|
||||||
|
//
|
||||||
|
// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command
|
||||||
|
// exits non-zero — but it says how many machines it did reach, because the old message ("nothing
|
||||||
|
// was sent") was the very claim that had become untrue.
|
||||||
|
func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||||
|
if err := couldNotBeResolved(nil, 3); err != nil {
|
||||||
|
t.Fatalf("a push that resolved every machine reported a problem: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a push that could not work out a machine reported success")
|
||||||
|
}
|
||||||
|
said := err.Error()
|
||||||
|
if strings.Contains(said, "nothing was sent") {
|
||||||
|
t.Errorf("the push says nothing was sent, and it sent two machines: %q", said)
|
||||||
|
}
|
||||||
|
for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} {
|
||||||
|
if !strings.Contains(said, want) {
|
||||||
|
t.Errorf("the refusal does not say %q: %q", want, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||||
|
func numbered() func(string) (int64, error) {
|
||||||
|
var n int64
|
||||||
|
return func(string) (int64, error) { n++; return n, nil }
|
||||||
|
}
|
||||||
@@ -0,0 +1,709 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||||
|
//
|
||||||
|
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
|
||||||
|
// may be and never settled — and changed through the controller: an ask cancelled, the queue
|
||||||
|
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
|
||||||
|
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
|
||||||
|
//
|
||||||
|
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
|
||||||
|
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
|
||||||
|
// rather than waiting for ever on an answer nobody will give.
|
||||||
|
|
||||||
|
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
|
||||||
|
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
|
||||||
|
const (
|
||||||
|
holderAsks = 15 * time.Second
|
||||||
|
killAnswer = 75 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||||
|
func dialTheBus() (*broker.JetStream, error) {
|
||||||
|
address, err := broker.BusAddress()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
js, err := broker.Dial(address)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||||
|
}
|
||||||
|
return js, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueCommand prints every ask in the build seat's work queue.
|
||||||
|
func queueCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if *asJSON {
|
||||||
|
body, err := json.MarshalIndent(q, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Print(queueText(q, time.Now()))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||||
|
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||||
|
func queueText(q link.Queue, now time.Time) string {
|
||||||
|
var b strings.Builder
|
||||||
|
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||||
|
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
|
||||||
|
ago := func(t time.Time) string {
|
||||||
|
if t.IsZero() {
|
||||||
|
return "asked at an unknown time"
|
||||||
|
}
|
||||||
|
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||||
|
}
|
||||||
|
what := func(a link.QueuedAsk) string {
|
||||||
|
s := a.Repository
|
||||||
|
if a.Path != "" {
|
||||||
|
s += " at " + a.Path
|
||||||
|
}
|
||||||
|
if a.Ref != "" {
|
||||||
|
s += " on " + a.Ref
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
if len(running) > 0 {
|
||||||
|
fmt.Fprintln(&b, "\nin flight:")
|
||||||
|
for _, a := range running {
|
||||||
|
where := "taken, not yet said where"
|
||||||
|
switch {
|
||||||
|
case a.On != "":
|
||||||
|
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||||
|
case a.Was != "":
|
||||||
|
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
fmt.Fprintln(&b, "\nwaiting:")
|
||||||
|
for _, a := range waiting {
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(dead) > 0 {
|
||||||
|
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||||
|
for _, a := range dead {
|
||||||
|
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case len(q.Asks) == 0:
|
||||||
|
fmt.Fprintln(&b, "nothing is asked of it")
|
||||||
|
default:
|
||||||
|
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelCommand drops one waiting or dead ask.
|
||||||
|
func cancelCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("cancel <build id>")
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ask, found := q.Find(args[0])
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
|
||||||
|
"what came of it", args[0], seat)
|
||||||
|
}
|
||||||
|
said, err := cancelAsk(ctx, js, open, seat, ask)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
|
||||||
|
//
|
||||||
|
// **In this order, and each step for a reason** (novox/hq ADR 0219):
|
||||||
|
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
|
||||||
|
// is running — that is `kill`, on the machine running it;
|
||||||
|
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
|
||||||
|
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
|
||||||
|
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
|
||||||
|
// read the mark first and ends it as cancelled, or is building it;
|
||||||
|
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
|
||||||
|
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
|
||||||
|
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
|
||||||
|
// that took it before the mark is building it, and only `kill` ends that;
|
||||||
|
// 5. the message is deleted by its sequence;
|
||||||
|
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
|
||||||
|
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
|
||||||
|
if ask.State == link.AskInFlight && ask.On != "" {
|
||||||
|
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
|
||||||
|
"ends the build where it runs", ask.ID, ask.On, ask.ID)
|
||||||
|
}
|
||||||
|
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
withdraw := func() {
|
||||||
|
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||||
|
switch ask.State {
|
||||||
|
case link.AskWaiting:
|
||||||
|
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||||
|
withdraw()
|
||||||
|
return "", err
|
||||||
|
} else if taken {
|
||||||
|
withdraw()
|
||||||
|
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
|
||||||
|
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
|
||||||
|
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
|
||||||
|
}
|
||||||
|
case link.AskInFlight:
|
||||||
|
if started, err := startedSince(ctx, js, seat, ask); err != nil {
|
||||||
|
withdraw()
|
||||||
|
return "", err
|
||||||
|
} else if started != "" {
|
||||||
|
withdraw()
|
||||||
|
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
|
||||||
|
"ends it there", ask.ID, started, ask.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||||
|
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||||
|
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||||
|
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||||
|
}
|
||||||
|
recordCancelled(ctx, open, ask)
|
||||||
|
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||||
|
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||||
|
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
|
||||||
|
var holderLooks = 5 * time.Second
|
||||||
|
|
||||||
|
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
|
||||||
|
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
|
||||||
|
// a start of a build.
|
||||||
|
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(holderLooks):
|
||||||
|
}
|
||||||
|
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||||
|
if !ask.AskedAt.IsZero() {
|
||||||
|
since = ask.AskedAt.Add(-time.Minute)
|
||||||
|
}
|
||||||
|
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
s, ok := heard.Started[ask.ID]
|
||||||
|
if !ok || heard.Outcomes[ask.ID] {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
at, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||||
|
if ask.Started.IsZero() || at.After(ask.Started) {
|
||||||
|
return s.On, nil
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// takenSince is whether the worker has handed out the ask at this sequence.
|
||||||
|
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
|
||||||
|
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||||
|
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
|
||||||
|
}
|
||||||
|
return info.Delivered.Stream >= seq, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
|
||||||
|
// then the plan that asked for it — by the id it asked with, or by repository and path.
|
||||||
|
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
|
||||||
|
r := ask.Request
|
||||||
|
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
|
||||||
|
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
|
||||||
|
_ = builds{open.inventory, open}.Built(ctx, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearCommand cancels every waiting ask, and with --dead every dead one too.
|
||||||
|
func clearCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("clear", flag.ContinueOnError)
|
||||||
|
dead := set.Bool("dead", false, "the dead asks too")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
said, err := clearQueue(ctx, js, open, seat, *dead)
|
||||||
|
fmt.Print(said)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
|
||||||
|
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
|
||||||
|
q, err := link.ReadQueue(ctx, js, seat)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
cancelled, refused := 0, 0
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
said, err := cancelAsk(ctx, js, open, seat, a)
|
||||||
|
if err != nil {
|
||||||
|
refused++
|
||||||
|
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cancelled++
|
||||||
|
fmt.Fprintf(&b, " %s\n", said)
|
||||||
|
}
|
||||||
|
what := "waiting"
|
||||||
|
if dead {
|
||||||
|
what = "waiting and dead"
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
|
||||||
|
if refused > 0 {
|
||||||
|
fmt.Fprintf(&b, ", %d could not be", refused)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(&b)
|
||||||
|
if n := len(q.Of(link.AskInFlight)); n > 0 {
|
||||||
|
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
|
||||||
|
}
|
||||||
|
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
|
||||||
|
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
|
||||||
|
}
|
||||||
|
if refused > 0 {
|
||||||
|
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
|
||||||
|
}
|
||||||
|
return b.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||||
|
// repository, path and ref — under a new id.
|
||||||
|
func rebuildCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("rebuild <module | build id>")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var source buildSource
|
||||||
|
var path, ref, module string
|
||||||
|
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||||
|
return err
|
||||||
|
} else if found {
|
||||||
|
source, module = sourceOfBuild(b, entries)
|
||||||
|
path, ref = b.Path, b.Ref
|
||||||
|
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||||
|
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||||
|
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||||
|
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||||
|
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||||
|
ref = followedBranch(e.Source.Ref)
|
||||||
|
follows := ref
|
||||||
|
if follows == "" {
|
||||||
|
follows = "the repository's default branch"
|
||||||
|
}
|
||||||
|
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||||
|
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||||
|
}
|
||||||
|
} else if e, known := entryNamed(entries, args[0]); known {
|
||||||
|
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||||
|
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||||
|
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||||
|
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||||
|
if module != "" {
|
||||||
|
joined, said, err := joinAPlan(ctx, open, module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if joined {
|
||||||
|
fmt.Println(said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id, err := askABuild(ctx, source, path, ref)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("rebuild asked as %s\n", id)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// entryNamed is the catalogue's entry for a module.
|
||||||
|
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.Manifest.Module == name {
|
||||||
|
return e, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return inventory.Entry{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||||
|
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||||
|
// (ADR 0111) — else the repository as it was cloned.
|
||||||
|
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
|
||||||
|
for _, e := range entries {
|
||||||
|
if (b.Module != "" && e.Manifest.Module == b.Module) ||
|
||||||
|
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
|
||||||
|
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return buildSource{Repository: b.Repository}, b.Module
|
||||||
|
}
|
||||||
|
|
||||||
|
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||||
|
func replayCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||||
|
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||||
|
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("replay <build id> [--register [--older]]")
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
b, found, err := inv.BuildByID(ctx, positionals[0])
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return fmt.Errorf("no build %s is recorded", positionals[0])
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
source, module := sourceOfBuild(b, entries)
|
||||||
|
var history []inventory.Build
|
||||||
|
if module != "" {
|
||||||
|
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||||
|
var outstanding []string
|
||||||
|
if *register {
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||||
|
js.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
plans, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(replaySaid(b, module, id, *register))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||||
|
//
|
||||||
|
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||||
|
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||||
|
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||||
|
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||||
|
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||||
|
// registered, unless --older says that is the point.
|
||||||
|
//
|
||||||
|
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||||
|
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||||
|
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||||
|
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||||
|
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||||
|
outstanding []string) error {
|
||||||
|
if b.Commit == "" {
|
||||||
|
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||||
|
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||||
|
}
|
||||||
|
if older && !register {
|
||||||
|
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||||
|
}
|
||||||
|
if register && len(outstanding) > 0 {
|
||||||
|
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||||
|
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||||
|
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||||
|
}
|
||||||
|
if !register || older {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, h := range history {
|
||||||
|
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||||
|
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||||
|
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||||
|
"without --register looks at it; --register --older registers it anyway",
|
||||||
|
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
|
||||||
|
func replaySaid(b inventory.Build, module, id string, register bool) string {
|
||||||
|
what := b.Repository
|
||||||
|
if module != "" {
|
||||||
|
what = module
|
||||||
|
}
|
||||||
|
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
|
||||||
|
if !register {
|
||||||
|
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
|
||||||
|
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
|
||||||
|
}
|
||||||
|
return said + "\n registered when it is built, as the module's current version — newer than every build " +
|
||||||
|
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
|
||||||
|
}
|
||||||
|
|
||||||
|
// killCommand finds the machine running a build and asks its holder to end it.
|
||||||
|
func killCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return errors.New("kill <build id>")
|
||||||
|
}
|
||||||
|
id := args[0]
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||||
|
if at, ok := link.BuildAskedAt(id); ok {
|
||||||
|
since = at.Add(-time.Minute)
|
||||||
|
}
|
||||||
|
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
started, ok := heard.Started[id]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
|
||||||
|
}
|
||||||
|
if heard.Outcomes[id] {
|
||||||
|
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
|
||||||
|
}
|
||||||
|
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return printHolderAnswer(started.On, answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
|
||||||
|
func pauseCommand(ctx context.Context, verb string, args []string) error {
|
||||||
|
if len(args) > 1 {
|
||||||
|
return fmt.Errorf("%s [node]", verb)
|
||||||
|
}
|
||||||
|
js, err := dialTheBus()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer js.Close()
|
||||||
|
seat := buildSeatHeld(ctx)
|
||||||
|
nodes := args
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(nodes) == 0 {
|
||||||
|
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failed := 0
|
||||||
|
for _, node := range nodes {
|
||||||
|
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("%s: %v\n", node, err)
|
||||||
|
failed++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := printHolderAnswer(node, answer); err != nil {
|
||||||
|
failed++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failed > 0 {
|
||||||
|
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
|
||||||
|
func printHolderAnswer(node string, answer link.Answer) error {
|
||||||
|
if answer.Error != "" {
|
||||||
|
fmt.Printf("%s: %s\n", node, answer.Error)
|
||||||
|
return errors.New(answer.Error)
|
||||||
|
}
|
||||||
|
var said struct {
|
||||||
|
Said string `json:"said"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
|
||||||
|
fmt.Printf("%s: %s\n", node, said.Said)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", node, string(answer.Result))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
|
||||||
|
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
entries, err := open.inventory.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return holdersAmong(entries, seat), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
|
||||||
|
func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []string
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.Manifest.ClaimsSeat(seat) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, n := range e.On {
|
||||||
|
if !seen[n] {
|
||||||
|
seen[n] = true
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||||
|
// by repository and path, and every open plan holding it not yet built.
|
||||||
|
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||||
|
var out []string
|
||||||
|
for _, a := range q.Asks {
|
||||||
|
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||||
|
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if module == "" {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
for _, p := range plans {
|
||||||
|
if !p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, tier := range p.Tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
if m == module {
|
||||||
|
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||||
|
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,772 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||||
|
//
|
||||||
|
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
|
||||||
|
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||||
|
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
|
||||||
|
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||||
|
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
|
||||||
|
|
||||||
|
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||||
|
func asksRecorded(t *testing.T) *[]string {
|
||||||
|
t.Helper()
|
||||||
|
var asked []string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
|
||||||
|
asked = append(asked, source.Repository+"#"+id)
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { askABuild = was })
|
||||||
|
return &asked
|
||||||
|
}
|
||||||
|
|
||||||
|
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
|
||||||
|
func twoTiers(t *testing.T, open *stores) {
|
||||||
|
t.Helper()
|
||||||
|
for _, name := range []string{"a", "b"} {
|
||||||
|
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
|
||||||
|
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
|
||||||
|
// tier, and when that build comes in the plan goes on and asks its next tier.
|
||||||
|
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
before := time.Now().UTC().Add(-time.Hour)
|
||||||
|
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a failed to build in tier 0",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||||
|
Why: link.KilledByHand}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := retryPlan(ctx, open, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := p.Modules["a"]
|
||||||
|
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
||||||
|
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, a.Build) {
|
||||||
|
t.Errorf("retry does not say the new id: %q", said)
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
||||||
|
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
||||||
|
}
|
||||||
|
asking, _ := link.BuildAskedAt(a.Build)
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
||||||
|
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
||||||
|
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 2 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What retry refuses, and says why.
|
||||||
|
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
plan := func(id, state string, created time.Time) inventory.Plan {
|
||||||
|
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
|
||||||
|
Created: created, State: state, Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||||
|
}
|
||||||
|
failed := plan("plan-1", inventory.PlanFailed, at)
|
||||||
|
for _, c := range []struct {
|
||||||
|
p inventory.Plan
|
||||||
|
others []inventory.Plan
|
||||||
|
says string
|
||||||
|
}{
|
||||||
|
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
|
||||||
|
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
|
||||||
|
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||||
|
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||||
|
} {
|
||||||
|
err := retryRefusal(c.p, c.others)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||||
|
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stopped := failed
|
||||||
|
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||||
|
stopped.Note = "a stopped at its first machine"
|
||||||
|
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||||
|
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||||
|
}
|
||||||
|
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
||||||
|
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||||
|
other.Branch = "release"
|
||||||
|
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
||||||
|
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
||||||
|
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||||
|
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
before := time.Now().UTC().Add(-time.Hour)
|
||||||
|
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a failed to build in tier 0",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||||
|
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||||
|
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||||
|
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 2 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||||
|
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||||
|
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||||
|
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||||
|
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||||
|
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||||
|
}
|
||||||
|
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||||
|
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||||
|
}
|
||||||
|
built := failed
|
||||||
|
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||||
|
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||||
|
t.Fatal("joined a plan that has the module built")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||||
|
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||||
|
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||||
|
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||||
|
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||||
|
history := []inventory.Build{newer, old}
|
||||||
|
|
||||||
|
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||||
|
t.Errorf("a dry run was refused: %v", err)
|
||||||
|
}
|
||||||
|
err := replayRefusal(old, "a", history, true, false, nil)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||||
|
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||||
|
t.Errorf("--register --older was refused: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||||
|
t.Errorf("registering the newest build again was refused: %v", err)
|
||||||
|
}
|
||||||
|
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||||
|
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||||
|
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||||
|
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||||
|
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||||
|
t.Error("a build that recorded no commit was replayed")
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||||
|
t.Error("--older without --register was taken")
|
||||||
|
}
|
||||||
|
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||||
|
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||||
|
q := link.Queue{Asks: []link.QueuedAsk{
|
||||||
|
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||||
|
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||||
|
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||||
|
}}
|
||||||
|
plans := []inventory.Plan{
|
||||||
|
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||||
|
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||||
|
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||||
|
}
|
||||||
|
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||||
|
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||||
|
t.Fatalf("outstanding: %v", outstanding)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||||
|
t.Errorf("registered over an outstanding ask: %v", err)
|
||||||
|
}
|
||||||
|
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||||
|
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||||
|
}
|
||||||
|
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||||
|
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||||
|
}
|
||||||
|
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||||
|
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||||
|
// paused on some holders only is not a reason the plan is waiting.
|
||||||
|
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
asked := now.Add(-12 * time.Minute)
|
||||||
|
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
|
||||||
|
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||||
|
|
||||||
|
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||||
|
line := planLineWith(p, now, all)
|
||||||
|
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a plan on a paused seat reads %q", line)
|
||||||
|
}
|
||||||
|
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||||
|
t.Errorf("status --json says %+v", st)
|
||||||
|
}
|
||||||
|
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||||
|
t.Errorf("a plan waiting on a paused seat counted late")
|
||||||
|
}
|
||||||
|
|
||||||
|
longAgo := now.Add(-25 * time.Hour)
|
||||||
|
forgotten := p
|
||||||
|
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||||
|
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a plan paused over a day reads %q", line)
|
||||||
|
}
|
||||||
|
|
||||||
|
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
|
||||||
|
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||||
|
t.Fatalf("%+v", some)
|
||||||
|
}
|
||||||
|
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||||
|
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||||
|
}
|
||||||
|
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||||
|
t.Errorf("status --json: %+v", st)
|
||||||
|
}
|
||||||
|
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||||
|
rolling := p
|
||||||
|
rolling.State = inventory.PlanRolling
|
||||||
|
if _, paused := pausedWaiting(rolling, all, now); paused {
|
||||||
|
t.Error("a rolling plan reads as waiting on the build seat")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The queue's verbs are the controller seat's, each to the command it names.
|
||||||
|
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
verb string
|
||||||
|
args map[string]any
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{"queue", nil, []string{"queue"}},
|
||||||
|
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||||
|
{"clear", nil, []string{"clear"}},
|
||||||
|
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||||
|
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||||
|
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||||
|
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||||
|
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||||
|
{"pause", nil, []string{"pause"}},
|
||||||
|
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||||
|
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
|
||||||
|
} {
|
||||||
|
got, err := argvFor(c.verb, c.args)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||||
|
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := argvFor("cancel", nil); err == nil {
|
||||||
|
t.Error("cancel without an id was taken")
|
||||||
|
}
|
||||||
|
declared := map[string]bool{}
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
declared[v.Name] = true
|
||||||
|
}
|
||||||
|
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
|
||||||
|
if !declared[v] {
|
||||||
|
t.Errorf("%s is not a verb of the controller seat", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- against a real bus -----------------------------------------------------------------------
|
||||||
|
|
||||||
|
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
|
||||||
|
// pointed at it, and a function that asks the seat one build.
|
||||||
|
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||||
|
t.Helper()
|
||||||
|
url := os.Getenv("MESH_TEST_NATS")
|
||||||
|
if url == "" {
|
||||||
|
t.Skip("MESH_TEST_NATS unset")
|
||||||
|
}
|
||||||
|
t.Setenv(broker.NATSVar, url)
|
||||||
|
js, err := broker.Dial(url)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(js.Close)
|
||||||
|
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
||||||
|
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
||||||
|
if err := broker.AssertMeshStreams(js); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||||
|
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
||||||
|
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||||
|
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
||||||
|
_ = js.Context().PurgeStream(broker.EventsStream)
|
||||||
|
})
|
||||||
|
ask := func(id, repository string) link.BuildRequest {
|
||||||
|
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
||||||
|
body, _ := json.Marshal(r)
|
||||||
|
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
return js, ask
|
||||||
|
}
|
||||||
|
|
||||||
|
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
|
||||||
|
func deadOne(t *testing.T, js *broker.JetStream) {
|
||||||
|
t.Helper()
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
for i := 0; i < worker.MaxDeliver; i++ {
|
||||||
|
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("delivery %d: %v", i+1, err)
|
||||||
|
}
|
||||||
|
_ = msgs[0].Nak()
|
||||||
|
}
|
||||||
|
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
|
||||||
|
// then, and stops counting it pending.
|
||||||
|
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
|
||||||
|
t.Fatalf("an ask past its deliveries was delivered again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
|
||||||
|
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
|
||||||
|
// is still found.
|
||||||
|
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
twoTiers(t, open)
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
asked, _ := link.BuildAskedAt(id)
|
||||||
|
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
||||||
|
t.Fatalf("the queue reads %+v", q)
|
||||||
|
}
|
||||||
|
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
||||||
|
strings.Contains(text, "secret-ish") {
|
||||||
|
t.Errorf("the queue says:\n%s", text)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := cancelCommand(ctx, []string{id}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||||
|
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
||||||
|
}
|
||||||
|
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
||||||
|
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
||||||
|
}
|
||||||
|
b, found, err := open.inventory.BuildByID(ctx, id)
|
||||||
|
if err != nil || !found || b.Failed != link.CancelledByHand {
|
||||||
|
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
||||||
|
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
||||||
|
}
|
||||||
|
if err := cancelCommand(ctx, []string{id}); err == nil {
|
||||||
|
t.Error("an ask cancelled already was cancelled again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
|
||||||
|
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
start := time.Now()
|
||||||
|
dead := link.NewBuildID(start)
|
||||||
|
ask(dead, "https://forge.example/novox/dead.git")
|
||||||
|
deadOne(t, js)
|
||||||
|
var waiting []string
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
||||||
|
waiting = append(waiting, id)
|
||||||
|
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
||||||
|
}
|
||||||
|
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
||||||
|
t.Fatalf("the queue reads %+v", q)
|
||||||
|
}
|
||||||
|
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
||||||
|
t.Errorf("clear said:\n%s", said)
|
||||||
|
}
|
||||||
|
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
||||||
|
t.Fatalf("after clear the queue is %+v", q.Asks)
|
||||||
|
}
|
||||||
|
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||||
|
t.Fatalf("clear --dead left %+v", q.Asks)
|
||||||
|
}
|
||||||
|
for _, id := range append(waiting, dead) {
|
||||||
|
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
||||||
|
t.Errorf("%s is recorded as %+v", id, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask in flight is not cancelled: kill ends it where it runs.
|
||||||
|
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, _ := q.Find(id)
|
||||||
|
if a.State != link.AskInFlight || a.On != "ace" {
|
||||||
|
t.Fatalf("the taken ask reads %+v", a)
|
||||||
|
}
|
||||||
|
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
||||||
|
t.Fatalf("an ask in flight was cancelled: %v", err)
|
||||||
|
}
|
||||||
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||||
|
t.Error("a refused cancel recorded an outcome")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
|
||||||
|
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
|
||||||
|
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||||
|
js, _ := aBuildQueue(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := map[string]string{}
|
||||||
|
handlers := map[string]link.ToolHandler{
|
||||||
|
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
var args struct{ ID string }
|
||||||
|
_ = json.Unmarshal(raw, &args)
|
||||||
|
asked["kill"] = args.ID
|
||||||
|
if args.ID != "build-running" {
|
||||||
|
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
||||||
|
}
|
||||||
|
return map[string]any{"said": "killed " + args.ID}, nil
|
||||||
|
},
|
||||||
|
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
asked["pause"] = "ace"
|
||||||
|
return map[string]any{"said": "ace is paused"}, nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer stop()
|
||||||
|
for _, id := range []string{"build-running", "build-other"} {
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asked["kill"] != "build-running" {
|
||||||
|
t.Fatalf("the holder was asked %v", asked)
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
||||||
|
t.Error("the holder's refusal was not the command's")
|
||||||
|
}
|
||||||
|
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
||||||
|
t.Errorf("a build nobody started: %v", err)
|
||||||
|
}
|
||||||
|
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
||||||
|
t.Fatalf("pause: %v %v", err, asked)
|
||||||
|
}
|
||||||
|
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
||||||
|
t.Error("a machine nothing answers on was resumed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||||
|
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||||
|
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
var sentTo [][]string
|
||||||
|
was := sendRollout
|
||||||
|
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||||
|
sentTo = append(sentTo, names)
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { sendRollout = was })
|
||||||
|
|
||||||
|
long := time.Now().UTC().Add(-2 * time.Hour)
|
||||||
|
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||||
|
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||||
|
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||||
|
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||||
|
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||||
|
t.Fatalf("retried under a newer plan: %v", err)
|
||||||
|
}
|
||||||
|
newer.State = inventory.PlanSuperseded
|
||||||
|
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
said, err := retryPlan(ctx, open, stopped.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
||||||
|
t.Fatalf("sent %v; said %q", sentTo, said)
|
||||||
|
}
|
||||||
|
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a := p.Modules["a"]
|
||||||
|
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
||||||
|
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
||||||
|
}
|
||||||
|
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
||||||
|
advancePlans(ctx, open)
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
||||||
|
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
||||||
|
}
|
||||||
|
if len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v", *asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
|
||||||
|
// the plan, asked later, never answers it (novox/hq ADR 0219).
|
||||||
|
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := time.Now().UTC().Add(-time.Minute)
|
||||||
|
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||||
|
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||||
|
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||||
|
if p.Modules["a"].State != "asked" {
|
||||||
|
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
// From the records too: a later build of the module recorded is not the plan's.
|
||||||
|
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
||||||
|
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
||||||
|
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
||||||
|
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
||||||
|
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rebuild of a build made at a commit asks what the module follows now, never the commit.
|
||||||
|
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
asked := asksRecorded(t)
|
||||||
|
twoTiers(t, open)
|
||||||
|
var refs []string
|
||||||
|
was := askABuild
|
||||||
|
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||||
|
refs = append(refs, ref)
|
||||||
|
return was(c, source, path, ref)
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||||
|
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||||
|
t.Fatalf("asked %v at %v", *asked, refs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
|
||||||
|
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
|
||||||
|
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||||
|
js, ask := aBuildQueue(t)
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
was := holderLooks
|
||||||
|
holderLooks = 300 * time.Millisecond
|
||||||
|
t.Cleanup(func() { holderLooks = was })
|
||||||
|
id := link.NewBuildID(time.Now())
|
||||||
|
ask(id, "https://forge.example/novox/a.git")
|
||||||
|
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||||
|
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = sub.Unsubscribe() }()
|
||||||
|
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, _ := q.Find(id)
|
||||||
|
if a.State != link.AskInFlight || a.On != "" {
|
||||||
|
t.Fatalf("the taken ask reads %+v", a)
|
||||||
|
}
|
||||||
|
// The holder that took it says it started, while the cancel waits.
|
||||||
|
go func() {
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||||
|
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
||||||
|
}()
|
||||||
|
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
||||||
|
t.Fatalf("a build that started was cancelled: %v", err)
|
||||||
|
}
|
||||||
|
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
||||||
|
t.Error("the withdrawn cancel is still marked")
|
||||||
|
}
|
||||||
|
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||||
|
t.Error("a withdrawn cancel recorded an outcome")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,268 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"sort"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The same answers, in a shape something other than a person can read.
|
||||||
|
//
|
||||||
|
// A board reads through interfaces and holds nothing (novox/hq 03-DESIGN/01-to-be/11-a-board.md).
|
||||||
|
// Everything it needs is already answered by these commands — as text, for people, which is not
|
||||||
|
// something a page can read. So each of them can say it again as JSON.
|
||||||
|
//
|
||||||
|
// **`--json` rather than a serving API**, because nothing needs one yet: whatever serves a board
|
||||||
|
// runs the command, and the constraint in the design holds either way — the board never touches a
|
||||||
|
// context's store. An API is the larger thing and should wait until something is asking for it.
|
||||||
|
//
|
||||||
|
// **These shapes are hard to change once anything is built against them.** So they stay close to
|
||||||
|
// what the domain already calls things, and carry no summary field that would have to be kept
|
||||||
|
// true. Nothing here is derived that a reader could not derive.
|
||||||
|
|
||||||
|
// meshStatus is what `status --json` says: the three questions, in the order they are asked.
|
||||||
|
type meshStatus struct {
|
||||||
|
// Wrong is every machine whose last declaration was refused or partly failed.
|
||||||
|
Wrong []machineDoing `json:"wrong"`
|
||||||
|
// Quiet is every machine not heard from lately. Not the same as wrong: new, switched off and
|
||||||
|
// unreachable are not "tried and could not".
|
||||||
|
Quiet []machineQuiet `json:"quiet"`
|
||||||
|
// Behind is every module built from something older than its source has.
|
||||||
|
Behind []moduleBehind `json:"behind"`
|
||||||
|
// Waiting is every machine not running what the mesh would send it. The same question as
|
||||||
|
// Behind one level down: that says the catalogue is old, this says a machine is — and only
|
||||||
|
// this one has somebody's change waiting inside it.
|
||||||
|
Waiting []machineWaiting `json:"waiting"`
|
||||||
|
// Reported is every machine's last word beside when it was last sent a declaration. A
|
||||||
|
// machine whose report is newer than its send has acted on the current declaration; one
|
||||||
|
// whose is older is still working — and Waiting cannot tell those apart, because the sent
|
||||||
|
// digest is recorded at send, not at apply.
|
||||||
|
Reported []machineReported `json:"reported"`
|
||||||
|
// Plans is what the last merges produced and where each stands (novox/hq ADR 0162): the
|
||||||
|
// open ones first, each saying its tier, what it waits for, and whether it has waited too long.
|
||||||
|
Plans []planStatus `json:"plans"`
|
||||||
|
// Unresolved is every machine that cannot be worked out at all, with what the mesh said when
|
||||||
|
// it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
|
||||||
|
// there is nothing to compare it against and nothing it can be behind — which is why a
|
||||||
|
// document without this field described a wholly blocked mesh as a well one.
|
||||||
|
//
|
||||||
|
// Per machine, and data. One node failing must never take the document away from a reader
|
||||||
|
// asking about the others.
|
||||||
|
Unresolved []machineUnresolved `json:"unresolved"`
|
||||||
|
// Network is why the private network could not be computed, when it could not; absent when it
|
||||||
|
// could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
|
||||||
|
// network, and a mesh whose hub is that node has no hub.
|
||||||
|
Network string `json:"network,omitempty"`
|
||||||
|
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
|
||||||
|
// "none at all".
|
||||||
|
Machines int `json:"machines"`
|
||||||
|
// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
|
||||||
|
Adopted []string `json:"adopted,omitempty"`
|
||||||
|
// Untaken is every module assigned to a machine that is holding what it found rather than
|
||||||
|
// running what the module declares, because nothing took it (novox/hq 04-ISSUES/125). Absent
|
||||||
|
// when nothing is held.
|
||||||
|
//
|
||||||
|
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||||
|
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||||
|
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||||
|
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||||
|
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||||
|
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||||
|
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||||
|
// what the mesh declared open.
|
||||||
|
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||||
|
// Unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||||
|
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||||
|
// dependency is met. Reported, not refused, until the switch.
|
||||||
|
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||||
|
// refuses traffic: where it is, whose the host reads it as, and what it refuses.
|
||||||
|
type machineFiltered struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Where string `json:"where"`
|
||||||
|
Owner string `json:"owner"`
|
||||||
|
Refuses string `json:"refuses"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// machineUntaken is one module a machine is holding rather than running, and how many resources of
|
||||||
|
// it are held.
|
||||||
|
type machineUntaken struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
// Held is how many of the module's resources the machine is keeping as it found them. Zero is
|
||||||
|
// impossible here: a module with nothing held is not in this list.
|
||||||
|
Held int `json:"held"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineUnresolved struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
// Problem is the mesh's own words, whole — newlines and all. It lists every requirement that
|
||||||
|
// could not be met, and a first line alone would name one of them and hide the rest.
|
||||||
|
Problem string `json:"problem"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineDoing struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
// Outcome is refused or failed. Kept distinct all the way out: they are fixed in different
|
||||||
|
// places, and one word for both sends half the readers to the wrong one.
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
Refused string `json:"refused,omitempty"`
|
||||||
|
Failed []struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
} `json:"failed,omitempty"`
|
||||||
|
Applied int `json:"applied"`
|
||||||
|
At time.Time `json:"at"`
|
||||||
|
// Since is when this same failure was first reported and Times how many reports in a row
|
||||||
|
// have said it; Stuck is the mesh's word for "enough of them" (novox/hq 04-ISSUES/065).
|
||||||
|
Since *time.Time `json:"since,omitempty"`
|
||||||
|
Times int `json:"times"`
|
||||||
|
Stuck bool `json:"stuck"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineReported struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Outcome string `json:"outcome"`
|
||||||
|
At *time.Time `json:"at,omitempty"`
|
||||||
|
Sent *time.Time `json:"sent,omitempty"`
|
||||||
|
// Current is whether the last report names the declaration last sent. Not derivable from
|
||||||
|
// the timestamps beside it: an apply begun under the previous declaration reports after the
|
||||||
|
// next send, newer and still about the old words.
|
||||||
|
Current bool `json:"current"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineWaiting struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
// Never is true when nothing has ever been sent to it. Not out of date: nobody has ever asked
|
||||||
|
// this machine to be anything, and the two read differently to whoever is looking.
|
||||||
|
Never bool `json:"never"`
|
||||||
|
Sent *time.Time `json:"sent,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type machineQuiet struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
// LastSeen is absent when the machine has never spoken, which is a different thing from
|
||||||
|
// having been quiet for a while.
|
||||||
|
LastSeen *time.Time `json:"lastSeen,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type moduleBehind struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
BuiltFrom string `json:"builtFrom"`
|
||||||
|
Head string `json:"head"`
|
||||||
|
On []string `json:"on"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusAsJSON answers the same questions as the text form, from the same reading.
|
||||||
|
//
|
||||||
|
// **It takes the whole reading rather than a growing argument list**, which is what let a new
|
||||||
|
// answer be added to the text form and forgotten here — the two are one function's output in two
|
||||||
|
// shapes, and they must not be able to differ about what was asked.
|
||||||
|
//
|
||||||
|
// It never fails on account of the mesh. Every per-machine problem in here is a field, so one
|
||||||
|
// machine that cannot be worked out cannot stop a caller reading about the others: a
|
||||||
|
// machine-readable interface that stops being machine-readable exactly when something is wrong is
|
||||||
|
// one nobody can build an alarm on.
|
||||||
|
func statusAsJSON(asked answers) ([]byte, error) {
|
||||||
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||||
|
behind, sources := asked.behind, asked.sources
|
||||||
|
waiting, reported := asked.waiting, asked.reported
|
||||||
|
|
||||||
|
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||||
|
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||||
|
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||||
|
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||||
|
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||||
|
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||||
|
for name := range asked.untaken {
|
||||||
|
untakenNodes = append(untakenNodes, name)
|
||||||
|
}
|
||||||
|
sort.Strings(untakenNodes)
|
||||||
|
for _, name := range untakenNodes {
|
||||||
|
modules := make([]string, 0, len(asked.untaken[name]))
|
||||||
|
for m := range asked.untaken[name] {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
for _, m := range modules {
|
||||||
|
out.Untaken = append(out.Untaken,
|
||||||
|
machineUntaken{Node: name, Module: m, Held: asked.untaken[name][m]})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
filteredNodes := make([]string, 0, len(asked.filtered))
|
||||||
|
for name := range asked.filtered {
|
||||||
|
filteredNodes = append(filteredNodes, name)
|
||||||
|
}
|
||||||
|
sort.Strings(filteredNodes)
|
||||||
|
for _, name := range filteredNodes {
|
||||||
|
f := asked.filtered[name]
|
||||||
|
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||||
|
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: "the found firewall",
|
||||||
|
Owner: inventory.FilterFoundFirewall, Refuses: fw.Kind + " is in force again"})
|
||||||
|
}
|
||||||
|
for _, x := range f.Others() {
|
||||||
|
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.Unheld = asked.unheld
|
||||||
|
for name := range asked.refused {
|
||||||
|
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||||
|
Node: name, Problem: asked.refused[name]})
|
||||||
|
}
|
||||||
|
sort.Slice(out.Unresolved, func(i, j int) bool {
|
||||||
|
return out.Unresolved[i].Node < out.Unresolved[j].Node
|
||||||
|
})
|
||||||
|
for _, r := range reported {
|
||||||
|
out.Reported = append(out.Reported, machineReported{
|
||||||
|
Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, m := range waiting {
|
||||||
|
out.Waiting = append(out.Waiting, machineWaiting{
|
||||||
|
Node: m.Node, Never: m.Never, Sent: m.SentAt})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, d := range wrong {
|
||||||
|
row := machineDoing{
|
||||||
|
Node: d.Node, Outcome: d.Outcome, Refused: d.Refused, Applied: d.Applied, At: d.At,
|
||||||
|
Since: d.Since, Times: d.Times, Stuck: d.Stuck(),
|
||||||
|
}
|
||||||
|
for _, f := range d.Failed {
|
||||||
|
row.Failed = append(row.Failed, struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}{ID: f.ID, Error: f.Error})
|
||||||
|
}
|
||||||
|
out.Wrong = append(out.Wrong, row)
|
||||||
|
}
|
||||||
|
for _, n := range quiet {
|
||||||
|
row := machineQuiet{Node: n.Name}
|
||||||
|
if !n.LastSeen.IsZero() {
|
||||||
|
seen := n.LastSeen
|
||||||
|
row.LastSeen = &seen
|
||||||
|
}
|
||||||
|
out.Quiet = append(out.Quiet, row)
|
||||||
|
}
|
||||||
|
for module, on := range behind {
|
||||||
|
from := sources[module]
|
||||||
|
out.Behind = append(out.Behind, moduleBehind{
|
||||||
|
Module: module, BuiltFrom: from.BuiltFrom, Head: from.Head, On: on,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return json.MarshalIndent(out, "", " ")
|
||||||
|
}
|
||||||
|
|
||||||
|
// say prints a value as JSON, for the commands that can answer either way.
|
||||||
|
func say(value any) error {
|
||||||
|
body, err := json.MarshalIndent(value, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,212 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The shape something other than a person reads.
|
||||||
|
//
|
||||||
|
// Hard to change once anything is built against it, so it stays close to what the domain already
|
||||||
|
// calls things and carries no summary field that would have to be kept true.
|
||||||
|
|
||||||
|
func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
|
||||||
|
behind map[string][]string, sources map[string]inventory.Source) map[string]any {
|
||||||
|
t.Helper()
|
||||||
|
body, err := statusAsJSON(answers{
|
||||||
|
wrong: wrong, nodes: nodes, quiet: quiet, behind: behind, sources: sources})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var parsed map[string]any
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
t.Fatalf("what a board would read is not JSON: %v", err)
|
||||||
|
}
|
||||||
|
return parsed
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRefusedAndFailedStayDistinctAllTheWayOut(t *testing.T) {
|
||||||
|
// They are fixed in different places, so one word for both would send half the readers of a
|
||||||
|
// page to the wrong one.
|
||||||
|
got := statusOf(t,
|
||||||
|
[]inventory.Doing{
|
||||||
|
{Node: "one", Outcome: inventory.OutcomeRefused, Refused: "a file needs a path"},
|
||||||
|
{Node: "two", Outcome: inventory.OutcomeFailed, Applied: 3,
|
||||||
|
Failed: []inventory.FailedResource{{ID: "shell.pkg", Error: "target not found"}}},
|
||||||
|
},
|
||||||
|
[]inventory.Node{{Name: "one"}, {Name: "two"}}, nil, nil, nil)
|
||||||
|
|
||||||
|
wrong, _ := got["wrong"].([]any)
|
||||||
|
if len(wrong) != 2 {
|
||||||
|
t.Fatalf("got %v", got["wrong"])
|
||||||
|
}
|
||||||
|
first, _ := wrong[0].(map[string]any)
|
||||||
|
if first["outcome"] != inventory.OutcomeRefused || first["refused"] == nil {
|
||||||
|
t.Fatalf("a refusal did not survive: %v", first)
|
||||||
|
}
|
||||||
|
second, _ := wrong[1].(map[string]any)
|
||||||
|
if second["outcome"] != inventory.OutcomeFailed {
|
||||||
|
t.Fatalf("a failure did not survive: %v", second)
|
||||||
|
}
|
||||||
|
if second["applied"] != float64(3) {
|
||||||
|
// "Three of eight" and "none of eight" are different machines.
|
||||||
|
t.Fatalf("what did apply was not carried: %v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMachineThatNeverSpokeSaysSoByOmission(t *testing.T) {
|
||||||
|
// Never heard from and quiet for a while are different situations, and a zero time would read
|
||||||
|
// as a date in 1970 on any page that formatted it.
|
||||||
|
got := statusOf(t, nil,
|
||||||
|
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
|
||||||
|
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
|
||||||
|
nil, nil)
|
||||||
|
|
||||||
|
quiet, _ := got["quiet"].([]any)
|
||||||
|
if len(quiet) != 2 {
|
||||||
|
t.Fatalf("got %v", got["quiet"])
|
||||||
|
}
|
||||||
|
never, _ := quiet[0].(map[string]any)
|
||||||
|
if _, said := never["lastSeen"]; said {
|
||||||
|
t.Fatalf("a machine that never spoke carries a time: %v", never)
|
||||||
|
}
|
||||||
|
away, _ := quiet[1].(map[string]any)
|
||||||
|
if _, said := away["lastSeen"]; !said {
|
||||||
|
t.Fatalf("a machine that has been quiet carries no time: %v", away)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNothingWrongIsAnEmptyListRatherThanNothing(t *testing.T) {
|
||||||
|
// A page distinguishing "no machines are wrong" from "this field is missing" would have to
|
||||||
|
// handle both, and null is the one that gets forgotten.
|
||||||
|
got := statusOf(t, nil, []inventory.Node{{Name: "a", LastSeen: time.Now()}}, nil, nil, nil)
|
||||||
|
for _, key := range []string{"wrong", "quiet", "behind"} {
|
||||||
|
list, ok := got[key].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("%q is %T, not a list", key, got[key])
|
||||||
|
}
|
||||||
|
if len(list) != 0 {
|
||||||
|
t.Fatalf("%q is not empty: %v", key, list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And how many machines there are, so a reader can tell "none wrong" from "none at all".
|
||||||
|
if got["machines"] != float64(1) {
|
||||||
|
t.Fatalf("got %v", got["machines"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWhatIsBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
|
||||||
|
// A module being out of date is a fact about the catalogue; machines running the old one is
|
||||||
|
// the thing with consequences.
|
||||||
|
got := statusOf(t, nil, nil, nil,
|
||||||
|
map[string][]string{"shell": {"workstation", "laptop"}},
|
||||||
|
map[string]inventory.Source{"shell": {BuiltFrom: "aaaaaaa1", Head: "bbbbbbb2"}})
|
||||||
|
|
||||||
|
behind, _ := got["behind"].([]any)
|
||||||
|
if len(behind) != 1 {
|
||||||
|
t.Fatalf("got %v", got["behind"])
|
||||||
|
}
|
||||||
|
row, _ := behind[0].(map[string]any)
|
||||||
|
if row["module"] != "shell" || row["builtFrom"] != "aaaaaaa1" || row["head"] != "bbbbbbb2" {
|
||||||
|
t.Fatalf("got %v", row)
|
||||||
|
}
|
||||||
|
on, _ := row["on"].([]any)
|
||||||
|
if len(on) != 2 {
|
||||||
|
t.Fatalf("the machines running the old one are not named: %v", row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoSecretIsInWhatABoardReads(t *testing.T) {
|
||||||
|
// Everything here comes from the mesh's own records, which hold no readable secret — but a
|
||||||
|
// shape a page is built against is exactly where one would eventually be added for
|
||||||
|
// convenience, so this says it out loud.
|
||||||
|
body, err := statusAsJSON(answers{
|
||||||
|
wrong: []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
|
||||||
|
Refused: "resource \"x\": a file needs a path"}},
|
||||||
|
nodes: []inventory.Node{{Name: "a"}},
|
||||||
|
refused: map[string]string{"a": "nothing provides \"database\", wanted by web"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, word := range []string{"password", "secret", "sealed", "credential", "token"} {
|
||||||
|
if strings.Contains(strings.ToLower(string(body)), word) {
|
||||||
|
t.Fatalf("what a board reads carries a %q field:\n%s", word, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failure that has been reported identically enough times is said to be stuck, with when it
|
||||||
|
// began and how many times — so a board can tell a machine looping on something that will never
|
||||||
|
// apply from one that failed a minute ago (novox/hq 04-ISSUES/065, ADR 0090).
|
||||||
|
func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
|
||||||
|
began := time.Date(2026, 9, 21, 9, 0, 0, 0, time.UTC)
|
||||||
|
got := statusOf(t,
|
||||||
|
[]inventory.Doing{
|
||||||
|
{Node: "looping", Outcome: inventory.OutcomeFailed, Since: &began, Times: inventory.StuckAfter,
|
||||||
|
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||||
|
{Node: "once", Outcome: inventory.OutcomeFailed, Since: &began, Times: 1,
|
||||||
|
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||||
|
},
|
||||||
|
[]inventory.Node{{Name: "looping"}, {Name: "once"}}, nil, nil, nil)
|
||||||
|
|
||||||
|
wrong, _ := got["wrong"].([]any)
|
||||||
|
looping, _ := wrong[0].(map[string]any)
|
||||||
|
if looping["stuck"] != true || looping["times"] != float64(inventory.StuckAfter) {
|
||||||
|
t.Fatalf("three identical failures are stuck: %v", looping)
|
||||||
|
}
|
||||||
|
if since, _ := looping["since"].(string); !strings.HasPrefix(since, "2026-09-21T09:00:00") {
|
||||||
|
t.Fatalf("when it began was not carried: %v", looping)
|
||||||
|
}
|
||||||
|
once, _ := wrong[1].(map[string]any)
|
||||||
|
if once["stuck"] != false || once["times"] != float64(1) {
|
||||||
|
t.Fatalf("one failure is not stuck: %v", once)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A converged machine something other than the mesh filters is named, per rule set, and is not
|
||||||
|
// well (novox/hq ADR 0168); one filtered by the mesh alone is not in the list.
|
||||||
|
func TestAMachineNotFilteredByTheMeshAloneIsNamedAndNotWell(t *testing.T) {
|
||||||
|
alone := inventory.Filtering{Filters: []inventory.Filter{
|
||||||
|
{Where: "table inet mesh, chain input", Owner: inventory.FilterMesh, Refuses: "policy drop"},
|
||||||
|
{Where: "table ip filter, chain DOCKER", Owner: inventory.FilterRuntime, Refuses: "drop"},
|
||||||
|
{Where: "table ip filter, chain f2b-sshd", Owner: inventory.FilterBan, Refuses: "ip saddr 192.0.2.1 reject"},
|
||||||
|
}}
|
||||||
|
if !alone.Alone() {
|
||||||
|
t.Fatal("the mesh's own, the runtime's and a ban are not the mesh alone")
|
||||||
|
}
|
||||||
|
notAlone := inventory.Filtering{
|
||||||
|
Filters: append(alone.Filters, inventory.Filter{Where: "chain HAL-MESH-ONLY (iptables-legacy)",
|
||||||
|
Owner: inventory.FilterOther, Refuses: `-A HAL-MESH-ONLY -m comment --comment "not public" -j DROP`}),
|
||||||
|
FoundFirewall: &inventory.FoundFirewall{Kind: "ufw", Active: true},
|
||||||
|
}
|
||||||
|
asked := answers{nodes: []inventory.Node{{Name: "home-server"}, {Name: "laptop"}},
|
||||||
|
filtered: map[string]inventory.Filtering{"home-server": notAlone}}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a machine not filtered by the mesh alone reads as well")
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var parsed struct {
|
||||||
|
Filtered []map[string]string `json:"filtered"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(parsed.Filtered) != 2 {
|
||||||
|
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||||
|
}
|
||||||
|
if parsed.Filtered[0]["node"] != "home-server" || parsed.Filtered[0]["owner"] != inventory.FilterFoundFirewall ||
|
||||||
|
parsed.Filtered[1]["where"] != "chain HAL-MESH-ONLY (iptables-legacy)" || parsed.Filtered[1]["owner"] != inventory.FilterOther {
|
||||||
|
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||||
|
}
|
||||||
|
if body, _ := statusAsJSON(answers{nodes: asked.nodes}); strings.Contains(string(body), `"filtered"`) {
|
||||||
|
t.Fatal("a mesh filtered by itself alone carries a filtered list")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What a build is recorded as, and what it is announced and handed on as.
|
||||||
|
//
|
||||||
|
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
|
||||||
|
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||||
|
// — and the manifest with those references in it. The mesh records the digest and the path
|
||||||
|
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
|
||||||
|
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
|
||||||
|
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
|
||||||
|
// rebuild of everything the mesh has ever built.
|
||||||
|
|
||||||
|
// recordedManifest is a build's manifest with the store's address taken off every reference the
|
||||||
|
// build itself made. Other references — an image a module runs from a public registry — are what
|
||||||
|
// they were, which is why this rewrites only what `made` names rather than everything that looks
|
||||||
|
// like an address.
|
||||||
|
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
|
||||||
|
announced := map[string]bool{}
|
||||||
|
for _, a := range made {
|
||||||
|
announced[a.Reference] = true
|
||||||
|
}
|
||||||
|
return withReferences(raw, func(ref string) (string, bool) {
|
||||||
|
if !announced[ref] {
|
||||||
|
return ref, false
|
||||||
|
}
|
||||||
|
return catalogue.Recorded(ref), true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
|
||||||
|
// composed into every reference the build made — recorded either way, before or after references
|
||||||
|
// were kept without their address.
|
||||||
|
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
|
||||||
|
recorded := map[string]bool{}
|
||||||
|
for _, a := range made {
|
||||||
|
recorded[catalogue.Recorded(a.Reference)] = true
|
||||||
|
}
|
||||||
|
return withReferences(raw, func(ref string) (string, bool) {
|
||||||
|
if !recorded[catalogue.Recorded(ref)] {
|
||||||
|
return ref, false
|
||||||
|
}
|
||||||
|
return catalogue.Rerouted(ref, address), true
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
|
||||||
|
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
|
||||||
|
// is, is the parser's to say, and it says so with a better sentence than anything here would.
|
||||||
|
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
var manifest map[string]any
|
||||||
|
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
resources, _ := manifest["resources"].([]any)
|
||||||
|
changed := false
|
||||||
|
for _, r := range resources {
|
||||||
|
resource, ok := r.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, key := range []string{"image", "source"} {
|
||||||
|
if written, ok := resource[key].(string); ok {
|
||||||
|
if rewritten, did := rewrite(written); did && rewritten != written {
|
||||||
|
resource[key] = rewritten
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !changed {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
out, err := json.Marshal(manifest)
|
||||||
|
if err != nil {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// routedArtifacts is a build's artifacts as something can fetch them now.
|
||||||
|
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
|
||||||
|
if address == "" {
|
||||||
|
return made
|
||||||
|
}
|
||||||
|
out := make([]inventory.Artifact, 0, len(made))
|
||||||
|
for _, a := range made {
|
||||||
|
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
|
||||||
|
Reference: catalogue.Rerouted(a.Reference, address)})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
|
||||||
|
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
|
||||||
|
// store's own node: loopback when nothing is on the network yet.
|
||||||
|
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if forNode == "" {
|
||||||
|
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
|
||||||
|
return "", err
|
||||||
|
} else if found {
|
||||||
|
forNode = holder
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return artifactStoreAddress(ctx, inv, shelf, forNode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
|
||||||
|
// when there is one, else loopback on the store's own node — when that node also holds a module
|
||||||
|
// requiring the store, which is what a builder is (genesis: one node holds both).
|
||||||
|
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
holder, _, found, err := artifactStoreHolder(ctx, inv)
|
||||||
|
if err != nil || !found {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, holder)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
besideIt := false
|
||||||
|
for _, a := range assigned {
|
||||||
|
for _, r := range shelf[a].Requires {
|
||||||
|
if r == catalogue.ArtifactStoreProvision {
|
||||||
|
besideIt = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !besideIt {
|
||||||
|
holder = ""
|
||||||
|
}
|
||||||
|
return artifactStoreAddress(ctx, inv, shelf, holder)
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,214 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it,
|
||||||
|
// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart.
|
||||||
|
func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{
|
||||||
|
// build dependencies: images on the runtime, a plugin on one of them
|
||||||
|
{From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||||
|
{From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||||
|
{From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared},
|
||||||
|
// a code dependency: the proxy packages the controller's source — same tier
|
||||||
|
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||||
|
{From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||||
|
// runtime dependencies: everything source-built is built by the builder
|
||||||
|
{From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||||
|
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
||||||
|
}
|
||||||
|
// The runtime image moved: everything on it, and what is built by what is on it.
|
||||||
|
set := reachableFrom([]string{"mesh-tools"}, edges)
|
||||||
|
// What stands on the runtime, and the builder that stands on it; not the controller, which the
|
||||||
|
// builder merely builds, nor the proxy that packages the controller.
|
||||||
|
want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"}
|
||||||
|
if len(set) != len(want) {
|
||||||
|
t.Fatalf("reachable from the runtime: %v, want %v", set, want)
|
||||||
|
}
|
||||||
|
tiers := tiersOf(set, edges)
|
||||||
|
pos := map[string]int{}
|
||||||
|
for i, tier := range tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
pos[m] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
||||||
|
t.Fatalf("the runtime then the builder: %v", tiers)
|
||||||
|
}
|
||||||
|
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
||||||
|
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
||||||
|
}
|
||||||
|
if pos["shop-plugin"] <= pos["shop"] {
|
||||||
|
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
||||||
|
}
|
||||||
|
if hasCycle(tiers, edges) {
|
||||||
|
t.Fatalf("no cycle here: %v", tiers)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The controller alone moved: the proxy with it, nothing else.
|
||||||
|
small := reachableFrom([]string{"mesh-controller"}, edges)
|
||||||
|
if len(small) != 3 {
|
||||||
|
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
||||||
|
}
|
||||||
|
// The builder packages the controller's source (same tier by that edge) and the controller is
|
||||||
|
// built by the builder (next tier by that one): the builder first, then the controller and the
|
||||||
|
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
|
||||||
|
smallTiers := tiersOf(small, edges)
|
||||||
|
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
|
||||||
|
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
|
||||||
|
}
|
||||||
|
// The builder alone moved: the builder, and nothing it builds.
|
||||||
|
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||||
|
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
||||||
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
||||||
|
p.Tier = pos["builder"]
|
||||||
|
rollsOut := func(m string) bool { return m == "builder" }
|
||||||
|
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
||||||
|
t.Fatalf("the builder gates the tier after it: %v", g)
|
||||||
|
}
|
||||||
|
p.Tier = 0
|
||||||
|
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
||||||
|
t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g)
|
||||||
|
}
|
||||||
|
if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 {
|
||||||
|
t.Fatalf("a module that only records its upgrade gates nothing: %v", g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A gate is open once every machine running the module has reported after it was built.
|
||||||
|
func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) {
|
||||||
|
built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC)
|
||||||
|
before, after := built.Add(-time.Minute), built.Add(time.Minute)
|
||||||
|
reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}}
|
||||||
|
ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports)
|
||||||
|
if ok || len(waiting) != 1 || waiting[0] != "home-server" {
|
||||||
|
t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting)
|
||||||
|
}
|
||||||
|
if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok {
|
||||||
|
t.Fatal("the machine that reported after the build holds the gate open")
|
||||||
|
}
|
||||||
|
if ok, _ := applied("builder", built, nil, reports); !ok {
|
||||||
|
t.Fatal("a module running nowhere gates nothing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cycle is not lost: what remains is one last tier, and the caller says so.
|
||||||
|
func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}}
|
||||||
|
tiers := tiersOf([]string{"a", "b"}, edges)
|
||||||
|
if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) {
|
||||||
|
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
||||||
|
// bundle a tier after the toolchain, not beside it.
|
||||||
|
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
||||||
|
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
||||||
|
[]string{"mesh-tools", "node-tools"}, edges)
|
||||||
|
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
||||||
|
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
||||||
|
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
||||||
|
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
||||||
|
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
||||||
|
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"mesh-controller": {State: "asked", AskedAt: &asked},
|
||||||
|
"builder": {State: "asked", AskedAt: &asked},
|
||||||
|
}}
|
||||||
|
records := map[string][]inventory.Build{
|
||||||
|
// Newest first, as Builds answers: the build after the ask is the outcome.
|
||||||
|
"mesh-controller": {
|
||||||
|
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
||||||
|
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
||||||
|
},
|
||||||
|
// Only a build from before the ask: not this ask's outcome.
|
||||||
|
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
||||||
|
}
|
||||||
|
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
||||||
|
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
||||||
|
}
|
||||||
|
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
||||||
|
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
||||||
|
}
|
||||||
|
if s := p.Modules["builder"]; s.State != "asked" {
|
||||||
|
t.Errorf("an ask with no record after it was settled: %+v", s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
||||||
|
// plan's late outcome — is not this ask's, built or failed.
|
||||||
|
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
||||||
|
late := map[string][]inventory.Build{"postgres": {
|
||||||
|
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
||||||
|
}}
|
||||||
|
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
||||||
|
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
||||||
|
}
|
||||||
|
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
||||||
|
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
||||||
|
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
||||||
|
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
||||||
|
r.Modules["postgres"].Commit != "4bcd5f73" {
|
||||||
|
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failure recorded after the ask fails the plan, as hearing it would have.
|
||||||
|
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
||||||
|
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
||||||
|
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
||||||
|
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
||||||
|
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
||||||
|
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
||||||
|
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
||||||
|
firstSent := built.Add(31 * time.Second)
|
||||||
|
firstReported := built.Add(43 * time.Second)
|
||||||
|
restSent := built.Add(58 * time.Second)
|
||||||
|
restReported := built.Add(74 * time.Second)
|
||||||
|
reports := []inventory.Reported{
|
||||||
|
{Node: "ace", At: &firstReported},
|
||||||
|
{Node: "g14", At: &restReported},
|
||||||
|
}
|
||||||
|
since := func(node string) time.Time {
|
||||||
|
if node == "ace" {
|
||||||
|
return firstSent
|
||||||
|
}
|
||||||
|
return restSent
|
||||||
|
}
|
||||||
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
||||||
|
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
||||||
|
}
|
||||||
|
// The old reading, every machine from the last send, is what held the plan.
|
||||||
|
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
||||||
|
t.Fatal("the single-moment reading should hold the first machine back")
|
||||||
|
}
|
||||||
|
early := built.Add(10 * time.Second)
|
||||||
|
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
||||||
|
t.Fatal("a report from before the machine was sent opened the gate")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,457 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/broker"
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
|
||||||
|
//
|
||||||
|
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
|
||||||
|
// transports and every one of them chooses by a single fact; this is the step that flips it. That
|
||||||
|
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
|
||||||
|
// bounded until here — and it means the useful work is almost all in the checking.
|
||||||
|
//
|
||||||
|
// So `rollout check` is the command that matters and the one that can be run any number of times
|
||||||
|
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
|
||||||
|
// `rollout` itself refuses unless the check is clean.
|
||||||
|
//
|
||||||
|
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
|
||||||
|
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
|
||||||
|
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
|
||||||
|
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
|
||||||
|
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
|
||||||
|
|
||||||
|
const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"
|
||||||
|
|
||||||
|
func rolloutCommand(ctx context.Context, args []string) error {
|
||||||
|
switch {
|
||||||
|
case len(args) == 1 && args[0] == "check":
|
||||||
|
return rolloutCheck(ctx)
|
||||||
|
case len(args) == 1 && args[0] == "mint":
|
||||||
|
return rolloutMint(ctx, false)
|
||||||
|
case len(args) == 2 && args[0] == "hand":
|
||||||
|
return rolloutHand(ctx, args[1])
|
||||||
|
case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
|
||||||
|
// Every credential minted afresh, whether or not one exists — for a mint that was wrong
|
||||||
|
// before anything was pushed. Afterwards nothing that received the old one still works,
|
||||||
|
// which is fine exactly when nothing received it.
|
||||||
|
return rolloutMint(ctx, true)
|
||||||
|
case len(args) == 1 && args[0] == "--confirm":
|
||||||
|
return errors.New(
|
||||||
|
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
|
||||||
|
"what is missing. Moving every node at once is the one step with nothing to inspect " +
|
||||||
|
"afterwards, so it is not being written before the check it depends on has been run " +
|
||||||
|
"against a real mesh")
|
||||||
|
default:
|
||||||
|
return errors.New(rolloutUsage)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolloutCheck says whether the mesh could move, and what would happen if it did.
|
||||||
|
func rolloutCheck(ctx context.Context) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
state, err := readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("the bus this mesh would move to")
|
||||||
|
if state.TheBus == "" {
|
||||||
|
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
|
||||||
|
} else {
|
||||||
|
standing := "not answering"
|
||||||
|
if state.ServerStanding {
|
||||||
|
standing = "answering"
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s — %s\n", state.TheBus, standing)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
fmt.Println("what would move")
|
||||||
|
for _, step := range broker.WhatMoves(state) {
|
||||||
|
fmt.Printf(" %s\n", step)
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
why := notReadyOf(state)
|
||||||
|
if len(why) == 0 {
|
||||||
|
fmt.Println("nothing is missing: this mesh could move its bus.")
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
|
||||||
|
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
|
||||||
|
for i, w := range why {
|
||||||
|
fmt.Printf(" %d. %s\n", i+1, w)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// readinessOf gathers what the mesh knows about its own ability to move.
|
||||||
|
//
|
||||||
|
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
|
||||||
|
// point: the answer is only useful if it can be had without committing to anything.
|
||||||
|
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
|
||||||
|
state := broker.Readiness{
|
||||||
|
Credentialled: map[string]bool{},
|
||||||
|
ModuleCredentialled: map[string]bool{},
|
||||||
|
// The old broker keeps its other clients on this installation, and saying so is how the plan
|
||||||
|
// stops reading as a retirement.
|
||||||
|
}
|
||||||
|
|
||||||
|
address, _, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
state.TheBus = address
|
||||||
|
if address != "" {
|
||||||
|
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
|
||||||
|
// to move onto a server that is not there should hear it here rather than afterwards.
|
||||||
|
//
|
||||||
|
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
|
||||||
|
// bus that requires TLS and a user fails at the handshake, and the check then reported a
|
||||||
|
// standing server as absent (seen live, 2026-09-28).
|
||||||
|
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||||
|
state.ServerStanding = true
|
||||||
|
js.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
nodes, err := inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
kept, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, n := range nodes {
|
||||||
|
state.Nodes = append(state.Nodes, n.Name)
|
||||||
|
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
|
||||||
|
state.Credentialled[n.Name] = has
|
||||||
|
|
||||||
|
assigned, err := inv.Assigned(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return state, err
|
||||||
|
}
|
||||||
|
for _, module := range assigned {
|
||||||
|
m, known := shelf[module]
|
||||||
|
if !known {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// The machine that holds the bus seat is the one that would be sent the user list.
|
||||||
|
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||||
|
state.Holder = n.Name
|
||||||
|
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
|
||||||
|
}
|
||||||
|
// A module that never speaks needs no credential, so it is not counted as missing one.
|
||||||
|
if !speaksOnTheBus(m) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
named := n.Name + "/" + module
|
||||||
|
state.Modules = append(state.Modules, named)
|
||||||
|
_, hasOne := kept[broker.Principal{
|
||||||
|
Kind: broker.KindModule, Node: n.Name, Module: module,
|
||||||
|
}.Username()]
|
||||||
|
state.ModuleCredentialled[named] = hasOne
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return state, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// speaksOnTheBus says whether a module reaches the bus at all.
|
||||||
|
//
|
||||||
|
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||||
|
// would bury the ones that matter under a list nobody can act on.
|
||||||
|
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||||
|
return len(m.Emits) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||||
|
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
|
||||||
|
// list became part of one.
|
||||||
|
//
|
||||||
|
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
|
||||||
|
// has still been sent it, and this question is about whether the mesh did its part.
|
||||||
|
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||||
|
digest, err := inv.Outstanding(ctx, node)
|
||||||
|
return err == nil && strings.TrimSpace(digest) != ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
|
||||||
|
// printing. The reasoning itself is the broker package's, where it is pure.
|
||||||
|
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
|
||||||
|
|
||||||
|
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
|
||||||
|
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
|
||||||
|
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
|
||||||
|
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
|
||||||
|
//
|
||||||
|
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
|
||||||
|
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
|
||||||
|
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
|
||||||
|
// the module that provides it is assigned, rather than read from this process's environment: this
|
||||||
|
// process is still on the old bus when this runs, and must be.
|
||||||
|
func rolloutMint(ctx context.Context, again bool) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
|
||||||
|
"must carry its fingerprint: %w", err)
|
||||||
|
}
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var busNode, controllerNode string
|
||||||
|
for _, e := range entries {
|
||||||
|
switch {
|
||||||
|
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
|
||||||
|
busNode = e.On[0]
|
||||||
|
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
|
||||||
|
controllerNode = e.On[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if busNode == "" {
|
||||||
|
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
|
||||||
|
"bus to mint credentials for — register and assign it first")
|
||||||
|
}
|
||||||
|
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
busHost := onNetwork[busNode]
|
||||||
|
if busHost == "" {
|
||||||
|
// **The hub is not in that map.** The machine that took over the tunnel is where the current
|
||||||
|
// bus already answers, and every machine dials it at the address the mesh handed them — so
|
||||||
|
// when the new bus runs on the same machine, that address is the one to tell them, with the
|
||||||
|
// new port. Found live: the control node is the hub, and the map lists the machines placed
|
||||||
|
// around it.
|
||||||
|
// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
|
||||||
|
// direction here — every URL built below adds its own) and no port.
|
||||||
|
_, _, host := broker.CredentialIn(known.Address)
|
||||||
|
if host == "" {
|
||||||
|
host = known.Address
|
||||||
|
}
|
||||||
|
if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
|
||||||
|
host = after
|
||||||
|
}
|
||||||
|
host = strings.TrimSpace(host)
|
||||||
|
if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
|
||||||
|
host = host[:i]
|
||||||
|
}
|
||||||
|
if host == "" {
|
||||||
|
return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
|
||||||
|
"current bus's address is unknown too, so no machine could be told where it is", busNode)
|
||||||
|
}
|
||||||
|
busHost = host
|
||||||
|
}
|
||||||
|
busAddress := busHost + ":4222"
|
||||||
|
|
||||||
|
records, err := inv.BusRecords(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
users, err := broker.Users(records)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
kept, err := inv.BusUsers(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
hashes := make(map[string]string, len(kept))
|
||||||
|
for name, u := range kept {
|
||||||
|
hashes[name] = u.PasswordHash
|
||||||
|
}
|
||||||
|
_, missing := broker.WithPasswords(users, hashes)
|
||||||
|
wanted := map[string]bool{}
|
||||||
|
for _, m := range missing {
|
||||||
|
wanted[m] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
var machines, modules, skipped int
|
||||||
|
for _, p := range users {
|
||||||
|
if !again && !wanted[p.Username()] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch p.Kind {
|
||||||
|
case broker.KindController:
|
||||||
|
if controllerNode == "" {
|
||||||
|
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
|
||||||
|
}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
|
||||||
|
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
|
||||||
|
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
|
||||||
|
}
|
||||||
|
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
|
||||||
|
|
||||||
|
case broker.KindNode:
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
membership, _ := json.Marshal(map[string]string{
|
||||||
|
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||||
|
})
|
||||||
|
key, err := inv.SealingKeyOf(ctx, p.Node)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
|
||||||
|
}
|
||||||
|
sealed, err := secrets.Seal(key, membership)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
machines++
|
||||||
|
|
||||||
|
case broker.KindModule, broker.KindNodeTools:
|
||||||
|
// The runtime is minted and delivered exactly as a module is (novox/hq ADR 0175): it is
|
||||||
|
// issued as the module it stands for, to that module's `broker` secret.
|
||||||
|
if p.Module == "mesh-controller" {
|
||||||
|
// The control plane is a module too, and its `broker` secret is the old bus's
|
||||||
|
// credential it is still using while this runs. Writing the new bus's blob there
|
||||||
|
// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
|
||||||
|
// is the controller principal's `bus` secret above; nothing else is needed here.
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m, inShelf := shelf[p.Module]
|
||||||
|
if !inShelf {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||||
|
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: busKindOf(p.Module), Node: p.Node, Module: p.Module})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
modules++
|
||||||
|
|
||||||
|
default:
|
||||||
|
skipped++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
|
||||||
|
machines, modules, skipped, busAddress)
|
||||||
|
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
|
||||||
|
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// providesBus is whether a manifest provides the mesh's bus.
|
||||||
|
func providesBus(m catalogue.Manifest) bool {
|
||||||
|
for _, o := range m.Provides {
|
||||||
|
if o.Name == "mesh-bus" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
|
||||||
|
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
|
||||||
|
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
|
||||||
|
// exists on this terminal and then only where it is written; the store keeps the hash, and the
|
||||||
|
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
|
||||||
|
func rolloutHand(ctx context.Context, node string) error {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
known, err := broker.FromEnvironment()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
|
||||||
|
}
|
||||||
|
busAddress, _, err := broker.OnNATS()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if busAddress == "" {
|
||||||
|
return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
|
||||||
|
}
|
||||||
|
_, _, bare := broker.CredentialIn(busAddress)
|
||||||
|
if _, after, has := strings.Cut(bare, "://"); has {
|
||||||
|
bare = after
|
||||||
|
}
|
||||||
|
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p := broker.Principal{Kind: broker.KindNode, Node: node}
|
||||||
|
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
membership, _ := json.Marshal(map[string]string{
|
||||||
|
"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||||
|
})
|
||||||
|
key, err := inv.SealingKeyOf(ctx, node)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sealed, err := secrets.Seal(key, membership)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// The one line of output is the membership itself, so it can be piped to the machine without
|
||||||
|
// being read on the way. Everything else goes to stderr.
|
||||||
|
fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
|
||||||
|
"then push the machine running the bus so the user list carries the new hash.\n",
|
||||||
|
node, catalogue.BusMembershipPath)
|
||||||
|
fmt.Println(string(membership))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
|
||||||
|
// once that machine has reported it applied; a module whose policy says together goes everywhere at
|
||||||
|
// once, as before.
|
||||||
|
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
|
||||||
|
running := []string{"novox", "ace", "g14"}
|
||||||
|
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
now := sentAt.Add(5 * time.Minute)
|
||||||
|
bound := 30 * time.Minute
|
||||||
|
after := sentAt.Add(time.Minute)
|
||||||
|
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
|
||||||
|
return nextRollout(s, running, together, reports, now, bound)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Together: every machine at once.
|
||||||
|
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
|
||||||
|
t.Fatalf("a together policy did not send every machine at once: %+v", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise the first by name, alone, when none has reported lately.
|
||||||
|
step := next(inventory.PlanModule{}, running, false, nil)
|
||||||
|
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
|
||||||
|
t.Fatalf("the first send was %+v, wanted ace alone", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||||
|
report := func(outcome string, current bool) []inventory.Reported {
|
||||||
|
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
|
||||||
|
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No report yet, or one about an older declaration than it was last sent: wait.
|
||||||
|
for what, reports := range map[string][]inventory.Reported{
|
||||||
|
"no report": nil,
|
||||||
|
"a report about older": report(inventory.OutcomeApplied, false),
|
||||||
|
} {
|
||||||
|
step := next(state, running, false, reports)
|
||||||
|
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
|
||||||
|
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Applied what it was last sent: the rest, and only the rest.
|
||||||
|
step = next(state, running, false, report(inventory.OutcomeApplied, true))
|
||||||
|
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
|
||||||
|
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Failed or refused: stop, the rest untouched.
|
||||||
|
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
|
||||||
|
step := next(state, running, false, report(outcome, true))
|
||||||
|
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
|
||||||
|
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
|
||||||
|
t.Errorf("a first machine that %s it: %+v", outcome, step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The machine holding the bus went with the first send: the rest wait for it too, and it is
|
||||||
|
// not sent again.
|
||||||
|
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
|
||||||
|
half := report(inventory.OutcomeApplied, true)
|
||||||
|
if step := next(both, running, false, half); step.waiting != "novox" {
|
||||||
|
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
|
||||||
|
}
|
||||||
|
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
|
||||||
|
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||||
|
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One machine, or none: nothing is waited for that cannot come.
|
||||||
|
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
|
||||||
|
t.Fatalf("a module nothing runs was sent: %+v", step)
|
||||||
|
}
|
||||||
|
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
|
||||||
|
t.Fatalf("a module on one machine waited for more: %+v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
|
||||||
|
// naming the machine and the bound; the rest are left alone.
|
||||||
|
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
|
||||||
|
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||||
|
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
|
||||||
|
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
|
||||||
|
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
|
||||||
|
t.Fatalf("a silent first machine: %+v", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The first machine is the first by name among those heard from lately: a laptop that is away is
|
||||||
|
// not the one the rest wait on. When none has been heard from, the first by name.
|
||||||
|
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
|
||||||
|
reports := []inventory.Reported{
|
||||||
|
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
|
||||||
|
}
|
||||||
|
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
|
||||||
|
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||||
|
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
|
||||||
|
// here as well put the bundle on every machine at once (novox/hq issue 249).
|
||||||
|
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
|
||||||
|
sent := time.Now()
|
||||||
|
plans := []inventory.Plan{
|
||||||
|
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
|
||||||
|
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
|
||||||
|
}
|
||||||
|
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
|
||||||
|
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
|
||||||
|
}
|
||||||
|
for _, m := range []string{"gitea", "keycloak"} {
|
||||||
|
if got := rolledOutByAPlan(plans, m); got != "" {
|
||||||
|
t.Errorf("%s, which no plan will send, was left to %s", m, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
|
||||||
|
// stopped at its first machine is not among them.
|
||||||
|
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
|
||||||
|
at := time.Now()
|
||||||
|
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
|
||||||
|
Modules: map[string]*inventory.PlanModule{
|
||||||
|
"agent": {State: "built"},
|
||||||
|
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
|
||||||
|
"sent": {State: "built", SentAt: &at},
|
||||||
|
"notes": {State: "built"},
|
||||||
|
"later": {},
|
||||||
|
}}
|
||||||
|
rollsOut := func(m string) bool { return m != "notes" }
|
||||||
|
sayUnsent(&p, rollsOut)
|
||||||
|
sayUnsent(&p, rollsOut)
|
||||||
|
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
|
||||||
|
t.Fatalf("the note reads %q", p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Whether a mesh could move its bus, read from a real store.
|
||||||
|
//
|
||||||
|
// The readiness reasoning has its own tests; this is about the gathering — that the question is
|
||||||
|
// answered from what the mesh actually holds, on a store with machines and modules in it, without
|
||||||
|
// writing anything.
|
||||||
|
|
||||||
|
func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
|
||||||
|
inv := inventory.ForTest(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
|
||||||
|
// module that never does.
|
||||||
|
for _, m := range []catalogue.Manifest{
|
||||||
|
{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||||
|
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
|
||||||
|
{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
|
||||||
|
{Module: "wallpaper", Version: "1"},
|
||||||
|
} {
|
||||||
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
|
||||||
|
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
state, err := readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if state.Holder != "anchor" {
|
||||||
|
t.Errorf("the machine holding the bus reads as %q", state.Holder)
|
||||||
|
}
|
||||||
|
if len(state.Nodes) != 2 {
|
||||||
|
t.Errorf("machines read as %v", state.Nodes)
|
||||||
|
}
|
||||||
|
// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
|
||||||
|
// never reaches the bus, and listing those would bury the ones that matter.
|
||||||
|
for _, m := range state.Modules {
|
||||||
|
if strings.HasSuffix(m, "/wallpaper") {
|
||||||
|
t.Errorf("a module that never speaks was counted: %v", state.Modules)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(state.Modules) != 2 {
|
||||||
|
t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
|
||||||
|
state.Modules)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing has been minted, so it is not ready — and it says so about each machine by name.
|
||||||
|
why := notReadyOf(state)
|
||||||
|
if len(why) == 0 {
|
||||||
|
t.Fatal("a mesh where nothing has a credential was reported ready to move")
|
||||||
|
}
|
||||||
|
said := strings.Join(why, "\n")
|
||||||
|
for _, name := range []string{"anchor", "laptop"} {
|
||||||
|
if !strings.Contains(said, name) {
|
||||||
|
t.Errorf("the refusal does not name %s: %s", name, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
|
||||||
|
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||||
|
Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
state, err = readinessOf(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !state.Credentialled["laptop"] {
|
||||||
|
t.Error("a machine that was minted a credential still reads as having none")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
|
||||||
|
// things, and only the first may be passed over when something is gathered across every machine
|
||||||
|
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
|
||||||
|
|
||||||
|
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{one.Module, two.Module} {
|
||||||
|
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err := planFor(t.Context(), open, "laptop")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("two modules claiming one seat composed anyway")
|
||||||
|
}
|
||||||
|
if !unresolvable(err) {
|
||||||
|
t.Fatalf("a set that cannot compose was not marked as the node's own problem: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
|
||||||
|
// Nothing is wrong with anchor. The question simply cannot be asked.
|
||||||
|
stopped, cancel := context.WithCancel(t.Context())
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
_, _, err := planFor(stopped, open, "anchor")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a plan composed against a store that could not be read")
|
||||||
|
}
|
||||||
|
if unresolvable(err) {
|
||||||
|
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
// rotateCommand replaces a credential and moves both ends together.
|
||||||
|
//
|
||||||
|
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
|
||||||
|
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
|
||||||
|
// password on every adoption and updated only the provider's row. Consumers on three nodes held
|
||||||
|
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
|
||||||
|
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
|
||||||
|
//
|
||||||
|
// Three things make that impossible here, and all three are deliberate:
|
||||||
|
//
|
||||||
|
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
|
||||||
|
// consumer's password touches one role and leaves every other consumer alone — and the list of who
|
||||||
|
// is affected is a query rather than an assumption.
|
||||||
|
//
|
||||||
|
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
|
||||||
|
// and left the sending to whoever remembered is the fault above, exactly.
|
||||||
|
//
|
||||||
|
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
|
||||||
|
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
|
||||||
|
// has half-rotated.
|
||||||
|
func rotateCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("rotate", flag.ContinueOnError)
|
||||||
|
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||||
|
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||||
|
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) != 1 {
|
||||||
|
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||||
|
}
|
||||||
|
provision := positionals[0]
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(holders) == 0 {
|
||||||
|
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||||
|
// and a rotation somebody believes happened is worse than one they know did not.
|
||||||
|
if *only != "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
|
||||||
|
"what it does hold", *only, provision, *only)
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
|
||||||
|
provision)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every machine at both ends, named before anything changes. A person about to rotate a
|
||||||
|
// production credential is entitled to know the blast radius before it is the past tense.
|
||||||
|
affected := map[string]bool{}
|
||||||
|
for _, h := range holders {
|
||||||
|
affected[h.Consumer] = true
|
||||||
|
affected[h.Provider] = true
|
||||||
|
}
|
||||||
|
machines := make([]string, 0, len(affected))
|
||||||
|
for name := range affected {
|
||||||
|
machines = append(machines, name)
|
||||||
|
}
|
||||||
|
sort.Strings(machines)
|
||||||
|
|
||||||
|
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
|
||||||
|
for _, h := range holders {
|
||||||
|
// The module, because a machine may hold several credentials for one provision and
|
||||||
|
// rotating "anchor's database password" now means rotating three of them.
|
||||||
|
fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, h := range holders {
|
||||||
|
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil {
|
||||||
|
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
|
||||||
|
// the remedy is to run this again rather than to repair anything — but a machine
|
||||||
|
// whose secret was discarded and not resent is holding a credential the provider is
|
||||||
|
// about to stop honouring, and that is worth knowing now.
|
||||||
|
return fmt.Errorf(
|
||||||
|
"rotating %s for %s on %s from %s: %w\n\nSome credentials were discarded and "+
|
||||||
|
"not yet sent. Run this again once the cause is fixed",
|
||||||
|
h.Provision, h.ConsumerModule, h.Consumer, h.Provider, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Both ends, in one send.** There is a window either way — a role's password changes on the
|
||||||
|
// provider and the file changes on the consumer, and they cannot be simultaneous — so the
|
||||||
|
// honest thing is to make it as short as the broker allows and to never leave it open across
|
||||||
|
// a command boundary, where it depends on somebody's memory.
|
||||||
|
fmt.Printf("\nsending to both ends:\n")
|
||||||
|
if err := sendTo(ctx, open, machines); err != nil {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
|
||||||
|
"Nothing on those machines has changed yet, so what is running keeps working "+
|
||||||
|
"until the provider next applies. Fix the cause and run `push --behind`", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
|
||||||
|
"changed cannot authenticate — `status` says who is still behind\n", len(machines))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||||
|
func asLocal(local string) string {
|
||||||
|
if local == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return " (as " + local + ")"
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
|
||||||
|
// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
|
||||||
|
// dependents, and `status` reports what composition does not yet refuse.
|
||||||
|
|
||||||
|
func serviceManagerHolder() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||||
|
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||||
|
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func packageManagerHolder() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "pacman", Version: "1",
|
||||||
|
Claims: []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
|
||||||
|
Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func aDaemon() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "sshd", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, packageManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
|
||||||
|
_, err := assign(ctx, open, "laptop", "sshd")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("sshd went onto a machine nothing holds the service manager of")
|
||||||
|
}
|
||||||
|
for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if contains(assigned, "sshd") {
|
||||||
|
t.Fatalf("a refused assignment was kept: %v", assigned)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The holders depend on each other, so neither goes on alone — and both go on in one act.
|
||||||
|
if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
|
||||||
|
t.Fatal("systemd went on alone though its package needs a package manager")
|
||||||
|
}
|
||||||
|
if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
|
||||||
|
t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
|
||||||
|
argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(argv, " ") != "assign laptop systemd pacman" {
|
||||||
|
t.Errorf("the seat's assign became %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, packageManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := unassign(ctx, open, "laptop", "systemd")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("the service manager came off a machine still running services")
|
||||||
|
}
|
||||||
|
for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the refusal does not name %q:\n%v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assigned, _ := open.inventory.Assigned(ctx, "laptop")
|
||||||
|
if !contains(assigned, "systemd") {
|
||||||
|
t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
|
||||||
|
}
|
||||||
|
if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatalf("a dependent could not come off: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
|
||||||
|
// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
|
||||||
|
defer catalogue.EnforcingSeatDependencies(false)()
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, serviceManagerHolder())
|
||||||
|
register(t, open, aDaemon())
|
||||||
|
// Assigned straight into the store: a machine whose modules predate the rule, which is every
|
||||||
|
// machine on the day it ships.
|
||||||
|
if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, refused := asked.refused["laptop"]; refused {
|
||||||
|
t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Error("a mesh with an unheld dependency reads as all well")
|
||||||
|
}
|
||||||
|
got := printed(t, func() error { return printStatus(asked) })
|
||||||
|
for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Errorf("status does not say %q:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Unheld []catalogue.Unheld `json:"unheld"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
|
||||||
|
t.Errorf("the document's unheld is %+v", doc.Unheld)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,293 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"text/tabwriter"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What this mesh can have one of, and who fills each (novox/hq ADR 0110).
|
||||||
|
//
|
||||||
|
// **Derived every time, never stored.** A seat is held by a module assignment, so the answer is
|
||||||
|
// computed from assignments by the same resolution that decides what every machine runs. A table
|
||||||
|
// of holders kept beside the assignments would be a second copy of one fact, and the first thing
|
||||||
|
// to be wrong about it.
|
||||||
|
|
||||||
|
// seatHolder is one assignment holding a seat.
|
||||||
|
type seatHolder struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||||
|
type seatRow struct {
|
||||||
|
Seat string `json:"seat"`
|
||||||
|
Scope string `json:"scope"`
|
||||||
|
Delivers string `json:"delivers,omitempty"`
|
||||||
|
Decision string `json:"decision"`
|
||||||
|
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||||
|
Replicated bool `json:"replicated,omitempty"`
|
||||||
|
Holders []seatHolder `json:"holders"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||||
|
// seat in the set.
|
||||||
|
//
|
||||||
|
// **The second list is not empty by construction.** Manifests are held to the set when they are
|
||||||
|
// registered, and a mesh can hold one registered before the set closed. Leaving its claim out of the
|
||||||
|
// overview would make the one thing the overview is for — what does this mesh have — quietly
|
||||||
|
// incomplete.
|
||||||
|
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
|
||||||
|
rows := make([]seatRow, 0, len(seats))
|
||||||
|
for _, s := range seats {
|
||||||
|
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||||
|
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||||
|
seen := map[seatHolder]bool{}
|
||||||
|
for _, h := range held {
|
||||||
|
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||||
|
// seat's former name groups under it after a rename (novox/hq ADR 0122).
|
||||||
|
hs, ok := catalogue.SeatNamed(h.Claim)
|
||||||
|
if !ok || hs.Name != s.Name || h.Scope != s.Scope {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
holder := seatHolder{Node: h.Node, Module: h.Module}
|
||||||
|
if !seen[holder] {
|
||||||
|
seen[holder] = true
|
||||||
|
row.Holders = append(row.Holders, holder)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(row.Holders, func(i, j int) bool {
|
||||||
|
if row.Holders[i].Node != row.Holders[j].Node {
|
||||||
|
return row.Holders[i].Node < row.Holders[j].Node
|
||||||
|
}
|
||||||
|
return row.Holders[i].Module < row.Holders[j].Module
|
||||||
|
})
|
||||||
|
rows = append(rows, row)
|
||||||
|
}
|
||||||
|
var outside []catalogue.Held
|
||||||
|
for _, h := range held {
|
||||||
|
// Outside the set only if it resolves to no seat at all — a former name still resolves.
|
||||||
|
if _, ok := catalogue.SeatNamed(h.Claim); !ok {
|
||||||
|
outside = append(outside, h)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(outside, func(i, j int) bool {
|
||||||
|
if outside[i].Claim != outside[j].Claim {
|
||||||
|
return outside[i].Claim < outside[j].Claim
|
||||||
|
}
|
||||||
|
return outside[i].Node < outside[j].Node
|
||||||
|
})
|
||||||
|
return rows, outside
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatCommand changes the set — the whole point of it being data (novox/hq ADR 0122) — and, since
|
||||||
|
// ADR 0131, changes who holds a seat.
|
||||||
|
func seatCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 3 && args[0] == "rename" {
|
||||||
|
from, to := args[1], args[2]
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
if err := open.inventory.RenameSeat(ctx, from, to); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is now %s — its former name still resolves, so nothing is rebuilt, "+
|
||||||
|
"re-registered or frozen (novox/hq ADR 0122)\n", from, to)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if len(args) == 3 && args[1] == "--to" {
|
||||||
|
return handOver(ctx, args[0], args[2], false)
|
||||||
|
}
|
||||||
|
if len(args) == 3 && args[1] == "--add" {
|
||||||
|
return handOver(ctx, args[0], args[2], true)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||||
|
"seat <name> --add <node>/<module>")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||||
|
// holder in between (novox/hq ADR 0131, design 28 task 5.3). The control plane finds its own bus
|
||||||
|
// through one of these seats; the day it was left empty mid-change is why this exists.
|
||||||
|
//
|
||||||
|
// Everything that could make the new holder wrong is refused here, before the row is written: the
|
||||||
|
// seat must exist, the assignment must exist, and the module must be able to hold the seat —
|
||||||
|
// claim it at its scope and provide what it delivers, judged against the store's row. What is
|
||||||
|
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||||
|
// and refusing to record a handover to a module the node has not started would make the handover
|
||||||
|
// impossible to do before the switch instead of as the switch.
|
||||||
|
//
|
||||||
|
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||||
|
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||||
|
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||||
|
// the one named, as it always did.
|
||||||
|
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||||
|
nodeName, module, ok := strings.Cut(to, "/")
|
||||||
|
if !ok || nodeName == "" || module == "" {
|
||||||
|
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
seat, known := catalogue.SeatNamed(seatName)
|
||||||
|
if !known {
|
||||||
|
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||||
|
}
|
||||||
|
if adding && !seat.Replicated {
|
||||||
|
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||||
|
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||||
|
}
|
||||||
|
assigned, err := inv.Assigned(ctx, nodeName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !slices.Contains(assigned, module) {
|
||||||
|
return fmt.Errorf("%s is not assigned to %s, so it cannot hold anything there — "+
|
||||||
|
"`assign %s %s` first", module, nodeName, nodeName, module)
|
||||||
|
}
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var m *catalogue.Manifest
|
||||||
|
for i := range entries {
|
||||||
|
if entries[i].Manifest.Module == module {
|
||||||
|
m = &entries[i].Manifest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m == nil {
|
||||||
|
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||||
|
}
|
||||||
|
var was string
|
||||||
|
var held []string
|
||||||
|
holdings, err := inv.Holdings(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, h := range holdings {
|
||||||
|
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||||
|
was = h.Node
|
||||||
|
held = append(held, h.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Recording who already holds the seat is not making a new holder, and is not judged like
|
||||||
|
// one.** On a mesh that predates the record, the first handover has to begin by writing down
|
||||||
|
// the standing holder — otherwise the next holder cannot be assigned beside it, because two
|
||||||
|
// eligible claimants with nothing on record are refused. That standing holder may no longer
|
||||||
|
// satisfy what the seat delivers (the row moved under it, on purpose, as ADR 0131's first step),
|
||||||
|
// and it holds regardless: derivation never read that column. So when nothing is on record and
|
||||||
|
// the named assignment is the one holding by derivation, only the claim itself is checked here.
|
||||||
|
// Every *change* of holder is judged in full.
|
||||||
|
claimsIt := false
|
||||||
|
for _, c := range m.Claims {
|
||||||
|
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||||
|
claimsIt = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if was == "" && claimsIt {
|
||||||
|
fmt.Printf("nothing was on record for %s; recording %s on %s as its standing holder\n",
|
||||||
|
seat.Name, module, nodeName)
|
||||||
|
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||||
|
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||||
|
}
|
||||||
|
if adding {
|
||||||
|
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||||
|
strings.Join(held, ", "))
|
||||||
|
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s is held by %s on %s\n", seat.Name, module, nodeName)
|
||||||
|
if was != "" && was != nodeName {
|
||||||
|
fmt.Printf(" `push %s` and `push %s` send both machines what changed\n", was, nodeName)
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" `push %s` sends the machine what changed; every other machine that reads the "+
|
||||||
|
"seat is re-declared by `push --behind`\n", nodeName)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func seatsCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("seats", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "the same, as JSON")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
shelf, err := inv.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Every node, none excluded: the same view of what each machine holds that planning uses.
|
||||||
|
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||||
|
|
||||||
|
if *asJSON {
|
||||||
|
out := struct {
|
||||||
|
Seats []seatRow `json:"seats"`
|
||||||
|
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||||
|
}{rows, outside}
|
||||||
|
body, err := json.MarshalIndent(out, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||||
|
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||||
|
for _, r := range rows {
|
||||||
|
delivers := r.Delivers
|
||||||
|
if delivers == "" {
|
||||||
|
delivers = "—"
|
||||||
|
}
|
||||||
|
holders := "unheld"
|
||||||
|
if len(r.Holders) > 0 {
|
||||||
|
parts := make([]string, 0, len(r.Holders))
|
||||||
|
for _, h := range r.Holders {
|
||||||
|
parts = append(parts, h.Module+" on "+h.Node)
|
||||||
|
}
|
||||||
|
holders = strings.Join(parts, ", ")
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||||
|
}
|
||||||
|
if err := w.Flush(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(outside) > 0 {
|
||||||
|
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||||
|
for _, h := range outside {
|
||||||
|
fmt.Printf(" %s %s on %s\n", h.Claim, h.Module, h.Node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The overview of what a mesh has (novox/hq ADR 0110).
|
||||||
|
|
||||||
|
func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) {
|
||||||
|
// An unheld seat is an answer — "this mesh has no forge" — so it is listed rather than omitted.
|
||||||
|
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||||
|
{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "postgres"},
|
||||||
|
})
|
||||||
|
if len(rows) != len(catalogue.Seats()) {
|
||||||
|
t.Fatalf("%d seats listed of %d", len(rows), len(catalogue.Seats()))
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
switch r.Seat {
|
||||||
|
case "mesh-store":
|
||||||
|
if len(r.Holders) != 1 || r.Holders[0].Module != "postgres" || r.Holders[0].Node != "anchor" {
|
||||||
|
t.Errorf("mesh-store is held by %+v", r.Holders)
|
||||||
|
}
|
||||||
|
if r.Delivers != "postgres-database" {
|
||||||
|
t.Errorf("mesh-store does not say what it delivers: %q", r.Delivers)
|
||||||
|
}
|
||||||
|
case "git":
|
||||||
|
if len(r.Holders) != 0 {
|
||||||
|
t.Errorf("git is held by %+v in a mesh with no forge", r.Holders)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
|
||||||
|
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||||
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
|
||||||
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||||
|
// Resolved twice, reported once: a machine is one holder however many passes saw it.
|
||||||
|
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||||
|
})
|
||||||
|
for _, r := range rows {
|
||||||
|
if r.Seat != "node-packet-filter" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
|
||||||
|
t.Fatalf("the packet filter is held by %+v", r.Holders)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.Fatal("the packet filter is not listed")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAClaimOutsideTheSetIsShownNotHidden(t *testing.T) {
|
||||||
|
// A manifest registered before the set closed can still hold one. Leaving it out would make
|
||||||
|
// the overview quietly incomplete, which is the one thing it may not be.
|
||||||
|
_, outside := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||||
|
{Claim: "the-controller", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "mesh-controller"},
|
||||||
|
})
|
||||||
|
if len(outside) != 1 || outside[0].Claim != "the-controller" {
|
||||||
|
t.Fatalf("a claim outside the set was not shown: %+v", outside)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,469 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"github.com/nats-io/nats.go/micro"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The mesh's own verbs, served as the mesh-controller seat's tools (novox/hq ADR 0154, design 33).
|
||||||
|
//
|
||||||
|
// **Each tool runs the command it names, in this same binary, and answers what it printed.** That is
|
||||||
|
// ADR 0035 taken literally: the logic lives once, in the command, and a surface is an adapter with no
|
||||||
|
// decisions in it. Running a fresh process rather than calling the function keeps two things true
|
||||||
|
// that calling it would not — every command opens and closes its own stores the way it does from a
|
||||||
|
// shell, and nothing a command prints to the process's standard output can leak into another call's
|
||||||
|
// answer. It also means a refusal is the same refusal in the same words, because it is the same
|
||||||
|
// output.
|
||||||
|
|
||||||
|
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
|
||||||
|
// command speaks JSON — the same as data.
|
||||||
|
type verbAnswer struct {
|
||||||
|
Output string `json:"output"`
|
||||||
|
OK bool `json:"ok"`
|
||||||
|
Answer any `json:"answer,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
||||||
|
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
||||||
|
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||||
|
str := func(key string) string {
|
||||||
|
v, _ := args[key].(string)
|
||||||
|
return strings.TrimSpace(v)
|
||||||
|
}
|
||||||
|
need := func(keys ...string) error {
|
||||||
|
for _, k := range keys {
|
||||||
|
if str(k) == "" {
|
||||||
|
return fmt.Errorf("%s needs %q", verb, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch verb {
|
||||||
|
case "command":
|
||||||
|
// The generic verb: the command line as given, split as a shell would split it, with
|
||||||
|
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
|
||||||
|
// binary and says so in its description (novox/hq ADR 0154, 0175).
|
||||||
|
if err := need("command"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv, err := splitCommandLine(str("command"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(argv) == 0 {
|
||||||
|
return nil, errors.New("command names no command")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "status":
|
||||||
|
return []string{"status", "--json"}, nil
|
||||||
|
case "nodes":
|
||||||
|
return []string{"node", "list", "--json"}, nil
|
||||||
|
case "node":
|
||||||
|
if err := need("node"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"node", "show", str("node")}, nil
|
||||||
|
case "modules":
|
||||||
|
return []string{"module", "list", "--json"}, nil
|
||||||
|
case "seats":
|
||||||
|
return []string{"seats", "--json"}, nil
|
||||||
|
case "builds":
|
||||||
|
if id := str("log"); id != "" {
|
||||||
|
return []string{"builds", "--log", id}, nil
|
||||||
|
}
|
||||||
|
if m := str("module"); m != "" {
|
||||||
|
return []string{"builds", m}, nil
|
||||||
|
}
|
||||||
|
return []string{"builds"}, nil
|
||||||
|
case "plans":
|
||||||
|
if r := str("repository"); r != "" {
|
||||||
|
argv := []string{"plans", "--what-if", r}
|
||||||
|
if p := str("paths"); p != "" {
|
||||||
|
argv = append(argv, "--paths", p)
|
||||||
|
}
|
||||||
|
if m := str("modules"); m != "" {
|
||||||
|
argv = append(argv, "--modules", m)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if id := str("stop"); id != "" {
|
||||||
|
return []string{"plans", "stop", id}, nil
|
||||||
|
}
|
||||||
|
if id := str("close"); id != "" {
|
||||||
|
return []string{"plans", "close", id}, nil
|
||||||
|
}
|
||||||
|
if id := str("retry"); id != "" {
|
||||||
|
return []string{"plans", "retry", id}, nil
|
||||||
|
}
|
||||||
|
if id := str("id"); id != "" {
|
||||||
|
return []string{"plans", id}, nil
|
||||||
|
}
|
||||||
|
return []string{"plans"}, nil
|
||||||
|
// The build queue (novox/hq ADR 0219).
|
||||||
|
case "queue":
|
||||||
|
return []string{"queue"}, nil
|
||||||
|
case "cancel", "kill":
|
||||||
|
if err := need("id"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{verb, str("id")}, nil
|
||||||
|
case "clear":
|
||||||
|
if str("dead") == "true" {
|
||||||
|
return []string{"clear", "--dead"}, nil
|
||||||
|
}
|
||||||
|
return []string{"clear"}, nil
|
||||||
|
case "rebuild":
|
||||||
|
if err := need("what"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"rebuild", str("what")}, nil
|
||||||
|
case "replay":
|
||||||
|
if err := need("id"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"replay", str("id")}
|
||||||
|
if str("register") == "true" {
|
||||||
|
argv = append(argv, "--register")
|
||||||
|
}
|
||||||
|
if str("older") == "true" {
|
||||||
|
argv = append(argv, "--older")
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "pause", "resume":
|
||||||
|
if n := str("node"); n != "" {
|
||||||
|
return []string{verb, n}, nil
|
||||||
|
}
|
||||||
|
return []string{verb}, nil
|
||||||
|
case "plan":
|
||||||
|
if err := need("node"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"plan", str("node"), "--json"}, nil
|
||||||
|
case "assign", "unassign":
|
||||||
|
if err := need("node", "module"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
||||||
|
// the seats that apply resources depend on each other and go on together.
|
||||||
|
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
||||||
|
case "pin":
|
||||||
|
if err := need("node", "provision", "from", "module"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"pin", str("node"), str("provision"), str("from"), str("module")}, nil
|
||||||
|
case "unpin":
|
||||||
|
if err := need("node", "provision"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"unpin", str("node"), str("provision")}, nil
|
||||||
|
case "push":
|
||||||
|
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||||
|
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||||
|
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||||
|
if n := str("node"); n != "" {
|
||||||
|
return []string{"push", n, "--wait", "0"}, nil
|
||||||
|
}
|
||||||
|
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||||
|
case "rotate":
|
||||||
|
if p := str("provision"); p != "" {
|
||||||
|
argv := []string{"rotate", p}
|
||||||
|
if c := str("consumer"); c != "" {
|
||||||
|
argv = append(argv, "--consumer", c)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
if str("node") != "" && str("module") != "" && str("secret") != "" {
|
||||||
|
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||||
|
}
|
||||||
|
// Half of either shape: the command says its usage, which names both shapes, and that is
|
||||||
|
// the answer the caller needs.
|
||||||
|
return []string{"rotate"}, nil
|
||||||
|
case "settings":
|
||||||
|
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||||
|
// because a tool has no file to hand the command; the command reads either.
|
||||||
|
if err := need("module"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
argv := []string{"settings", "set", str("module")}
|
||||||
|
switch {
|
||||||
|
case str("clear") == "true":
|
||||||
|
argv = []string{"settings", "clear", str("module")}
|
||||||
|
case str("values") != "":
|
||||||
|
argv = append(argv, str("values"))
|
||||||
|
}
|
||||||
|
// Neither values nor clear: the command says its usage, which names both, and that is the
|
||||||
|
// answer the caller needs — the same as `rotate` given half of either shape.
|
||||||
|
if n := str("node"); n != "" {
|
||||||
|
argv = append(argv, "--node", n)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
case "issue":
|
||||||
|
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
|
||||||
|
// into the mesh's records and delivered at the machine's next push, which is the caller's to
|
||||||
|
// ask for — so the mesh is never pushed as a side effect of a credential.
|
||||||
|
if err := need("node", "module"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
|
||||||
|
case "build":
|
||||||
|
if err := need("repository"); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
|
||||||
|
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
|
||||||
|
// repository given without a scheme is a path on the forge holding the git seat.
|
||||||
|
argv := []string{"build", str("repository"), "--wait", "0"}
|
||||||
|
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
|
||||||
|
argv = append(argv, "--self")
|
||||||
|
}
|
||||||
|
if p := str("path"); p != "" {
|
||||||
|
argv = append(argv, "--path", p)
|
||||||
|
}
|
||||||
|
if r := str("ref"); r != "" {
|
||||||
|
argv = append(argv, "--ref", r)
|
||||||
|
}
|
||||||
|
return argv, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||||
|
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||||
|
|
||||||
|
// runVerb runs this binary with the given command line and gathers what it said.
|
||||||
|
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||||
|
self, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return verbAnswer{}, err
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, self, argv...)
|
||||||
|
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||||
|
// from a shell in this container would have, because it is that.
|
||||||
|
cmd.Env = os.Environ()
|
||||||
|
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||||
|
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||||
|
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||||
|
// nothing (2026-09-30, the first status asked through the console had no `answer`).
|
||||||
|
var stdout, stderr bytes.Buffer
|
||||||
|
cmd.Stdout = &stdout
|
||||||
|
cmd.Stderr = &stderr
|
||||||
|
runErr := cmd.Run()
|
||||||
|
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
|
||||||
|
if jsonVerbs[argv[0]] && runErr == nil {
|
||||||
|
var parsed any
|
||||||
|
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
|
||||||
|
answer.Answer = parsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var exit *exec.ExitError
|
||||||
|
if runErr != nil && !errors.As(runErr, &exit) {
|
||||||
|
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||||
|
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
||||||
|
}
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||||
|
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||||
|
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||||
|
// would take the whole control plane down for one word (novox/hq ADR 0185).
|
||||||
|
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||||
|
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||||
|
if !known {
|
||||||
|
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||||
|
}
|
||||||
|
var behind []string
|
||||||
|
handlers := map[string]link.ToolHandler{}
|
||||||
|
for _, v := range seat.Serves {
|
||||||
|
verb := v.Name
|
||||||
|
if verb == "tools" {
|
||||||
|
handlers[verb] = func(ctx context.Context, _ json.RawMessage) (any, error) {
|
||||||
|
return seatTools(), nil
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
||||||
|
// **A row ahead of this binary is not a reason to go silent.**
|
||||||
|
//
|
||||||
|
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||||
|
// this build does not know means the row was widened by a newer one — the ordinary
|
||||||
|
// state of a roll-out, and of a push that put an older control plane back. Refusing to
|
||||||
|
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
|
||||||
|
// mesh off the bus for ten minutes, and the way back was a human running the binary by
|
||||||
|
// hand, because the thing that would have repaired it is the thing that was down
|
||||||
|
// (novox/hq 04-ISSUES/201, ADR 0185).
|
||||||
|
//
|
||||||
|
// So the verbs this binary knows are served, and this one answers the reason instead of
|
||||||
|
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
|
||||||
|
// — including the push that replaces this binary with the one whose verb it is.
|
||||||
|
behind = append(behind, verb)
|
||||||
|
reason := err
|
||||||
|
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
|
||||||
|
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
|
||||||
|
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
|
||||||
|
verb, catalogue.ControllerSeatName, reason)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||||
|
args := map[string]any{}
|
||||||
|
if len(raw) > 0 {
|
||||||
|
if err := json.Unmarshal(raw, &args); err != nil {
|
||||||
|
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
argv, err := argvFor(verb, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return runVerb(ctx, argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return handlers, behind, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||||
|
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
||||||
|
func seatTools() map[string]any {
|
||||||
|
var seats []map[string]any
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
if len(s.Serves) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var tools []map[string]any
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
tools = append(tools, map[string]any{
|
||||||
|
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
||||||
|
}
|
||||||
|
return map[string]any{"seats": seats}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
||||||
|
// verb runnable rather than that it happens to want the arguments the check guessed.
|
||||||
|
func sampleArguments(v catalogue.Verb) map[string]any {
|
||||||
|
sample := map[string]any{"node": "x", "module": "x", "repository": "x"}
|
||||||
|
switch required := v.Input["required"].(type) {
|
||||||
|
case []string:
|
||||||
|
for _, k := range required {
|
||||||
|
sample[k] = "x"
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
for _, k := range required {
|
||||||
|
if name, ok := k.(string); ok {
|
||||||
|
sample[name] = "x"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sample
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
|
||||||
|
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
|
||||||
|
// escapes the next character inside double quotes or outside any. No expansion of anything.
|
||||||
|
func splitCommandLine(line string) ([]string, error) {
|
||||||
|
var words []string
|
||||||
|
var cur strings.Builder
|
||||||
|
inWord := false
|
||||||
|
quote := rune(0)
|
||||||
|
runes := []rune(line)
|
||||||
|
for i := 0; i < len(runes); i++ {
|
||||||
|
r := runes[i]
|
||||||
|
switch {
|
||||||
|
case quote == '\'':
|
||||||
|
if r == '\'' {
|
||||||
|
quote = 0
|
||||||
|
} else {
|
||||||
|
cur.WriteRune(r)
|
||||||
|
}
|
||||||
|
case quote == '"':
|
||||||
|
if r == '"' {
|
||||||
|
quote = 0
|
||||||
|
} else if r == '\\' && i+1 < len(runes) {
|
||||||
|
i++
|
||||||
|
cur.WriteRune(runes[i])
|
||||||
|
} else {
|
||||||
|
cur.WriteRune(r)
|
||||||
|
}
|
||||||
|
case r == '\'' || r == '"':
|
||||||
|
quote = r
|
||||||
|
inWord = true
|
||||||
|
case r == '\\' && i+1 < len(runes):
|
||||||
|
i++
|
||||||
|
cur.WriteRune(runes[i])
|
||||||
|
inWord = true
|
||||||
|
case r == ' ' || r == '\t' || r == '\n':
|
||||||
|
if inWord {
|
||||||
|
words = append(words, cur.String())
|
||||||
|
cur.Reset()
|
||||||
|
inWord = false
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
cur.WriteRune(r)
|
||||||
|
inWord = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if quote != 0 {
|
||||||
|
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||||
|
}
|
||||||
|
if inWord {
|
||||||
|
words = append(words, cur.String())
|
||||||
|
}
|
||||||
|
return words, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||||
|
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||||
|
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||||
|
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||||
|
about := map[string]catalogue.Verb{}
|
||||||
|
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||||
|
if s.Name == catalogue.ControllerSeatName {
|
||||||
|
for _, v := range s.Serves {
|
||||||
|
about[v.Name] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
verbs := make([]string, 0, len(handlers))
|
||||||
|
for verb := range handlers {
|
||||||
|
verbs = append(verbs, verb)
|
||||||
|
}
|
||||||
|
sort.Strings(verbs)
|
||||||
|
var endpoints []micro.EndpointInfo
|
||||||
|
for _, verb := range verbs {
|
||||||
|
schema, _ := json.Marshal(about[verb].Input)
|
||||||
|
// The same shape every tool runtime announces in (node-tools' announce package): the name is
|
||||||
|
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
|
||||||
|
endpoints = append(endpoints, micro.EndpointInfo{
|
||||||
|
Name: catalogue.ControllerSeatName + "__" + verb,
|
||||||
|
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
|
||||||
|
QueueGroup: "seat." + catalogue.ControllerSeatName,
|
||||||
|
Metadata: map[string]string{
|
||||||
|
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
|
||||||
|
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
|
||||||
|
"description": about[verb].Description, "schema": string(schema),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return micro.Info{
|
||||||
|
ServiceIdentity: micro.ServiceIdentity{
|
||||||
|
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
|
||||||
|
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
|
||||||
|
},
|
||||||
|
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
|
||||||
|
Endpoints: endpoints,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,298 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every verb the mesh-controller seat declares is one this binary can run, with the arguments the
|
||||||
|
// schema names and no other (novox/hq ADR 0154, ADR 0035).
|
||||||
|
func TestEveryDeclaredVerbHasACommandLine(t *testing.T) {
|
||||||
|
for _, v := range catalogue.ControllerVerbs {
|
||||||
|
if v.Name == "tools" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
args := map[string]any{}
|
||||||
|
props, _ := v.Input["properties"].(map[string]any)
|
||||||
|
for name := range props {
|
||||||
|
args[name] = "x"
|
||||||
|
}
|
||||||
|
argv, err := argvFor(v.Name, args)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("%s: %v", v.Name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if argv[0] == "" {
|
||||||
|
t.Errorf("%s: empty command", v.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
||||||
|
// (novox/hq ADR 0157).
|
||||||
|
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||||
|
argv, err := argvFor("builds", map[string]any{"log": "build-17"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(argv, " ") != "builds --log build-17" {
|
||||||
|
t.Fatalf("builds with a log id became %q", strings.Join(argv, " "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The build tool takes a repository as a URL or as its path on the forge holding the git seat, and
|
||||||
|
// says which it was given, so the command reads the path as a seat source rather than handing it to
|
||||||
|
// git as written (novox/hq issue 176). And it never waits: the id follows the build.
|
||||||
|
func TestTheBuildToolTellsAForgePathFromAURL(t *testing.T) {
|
||||||
|
argv, _ := argvFor("build", map[string]any{"repository": "novox/mesh-catalog", "path": "modules/x"})
|
||||||
|
if line := strings.Join(argv, " "); !strings.Contains(line, "--self") || !strings.Contains(line, "--wait 0") {
|
||||||
|
t.Fatalf("a forge path is a seat source, not waited for; got %q", line)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("build", map[string]any{"repository": "https://example.tld/o/r.git"})
|
||||||
|
if line := strings.Join(argv, " "); strings.Contains(line, "--self") {
|
||||||
|
t.Fatalf("a URL is cloned as given; got %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `rotate` is one verb with two shapes (ADR 0114, issue 180): a pair credential by provision, or a
|
||||||
|
// module's own secret by machine, module and name.
|
||||||
|
func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||||
|
argv, _ := argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace"})
|
||||||
|
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||||
|
t.Fatalf("a pair credential: %v", argv)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||||
|
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||||
|
t.Fatalf("an own secret: %v", argv)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("rotate", map[string]any{"node": "ace"})
|
||||||
|
if strings.Join(argv, " ") != "rotate" {
|
||||||
|
t.Fatalf("half an own secret falls to the command's usage: %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
|
||||||
|
func TestSettingsSetsOrClearsALayer(t *testing.T) {
|
||||||
|
argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != `settings set dnsmasq {"a":1} --node ace` {
|
||||||
|
t.Fatalf("set on a machine: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq", "clear": "true"})
|
||||||
|
if strings.Join(argv, " ") != "settings clear dnsmasq" {
|
||||||
|
t.Fatalf("clear for the mesh: %v", argv)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"})
|
||||||
|
if strings.Join(argv, " ") != "settings set dnsmasq" {
|
||||||
|
t.Fatalf("a set with no values falls to the command's usage: %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `issue` is `module issue` at a shell: the module and the machine, and nothing that would push. A
|
||||||
|
// module's bus account was mintable only from the controller's command line, so an agent working
|
||||||
|
// through the tools could not finish a rollout that gave a module one (novox/hq issue 191).
|
||||||
|
func TestIssueGivesAModuleItsAccountOnAMachine(t *testing.T) {
|
||||||
|
argv, err := argvFor("issue", map[string]any{"node": "ace", "module": "route-proxy"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Join(argv, " ") != "module issue route-proxy --node ace" {
|
||||||
|
t.Fatalf("issue runs %v", argv)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("issue", map[string]any{"module": "route-proxy"}); err == nil {
|
||||||
|
t.Error("an account was issued without saying which machine reads it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A required argument missing is refused in the verb's own words, before anything runs.
|
||||||
|
func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||||
|
if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) {
|
||||||
|
t.Fatalf("node without a machine was accepted: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("upgrade", map[string]any{}); err == nil {
|
||||||
|
t.Fatal("a verb the seat does not serve was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||||
|
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||||
|
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||||
|
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||||
|
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||||
|
t.Fatalf("push waits: %v", argv)
|
||||||
|
}
|
||||||
|
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||||
|
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
|
||||||
|
t.Fatalf("build: %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// What `tools` answers is the seats' records, with each verb's schema.
|
||||||
|
func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
||||||
|
handlers, behind, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(behind) != 0 {
|
||||||
|
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||||
|
}
|
||||||
|
if len(handlers) != len(catalogue.ControllerVerbs) {
|
||||||
|
t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
|
||||||
|
}
|
||||||
|
answer := seatTools()
|
||||||
|
seats, _ := answer["seats"].([]map[string]any)
|
||||||
|
var found bool
|
||||||
|
for _, s := range seats {
|
||||||
|
if s["seat"] == catalogue.ControllerSeatName {
|
||||||
|
found = true
|
||||||
|
tools, _ := s["tools"].([]map[string]any)
|
||||||
|
if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil {
|
||||||
|
t.Fatalf("the controller seat's tools are not listed in full: %v", tools)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatal("the mesh-controller seat is not in the listing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
|
||||||
|
// printed beside the document does not take the document away. The test binary stands in for the
|
||||||
|
// controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests.
|
||||||
|
func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) {
|
||||||
|
jsonVerbs["-test.run"] = true
|
||||||
|
t.Cleanup(func() { delete(jsonVerbs, "-test.run") })
|
||||||
|
answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !answer.OK {
|
||||||
|
t.Fatalf("the command failed: %s", answer.Output)
|
||||||
|
}
|
||||||
|
if !strings.Contains(answer.Output, "PASS") {
|
||||||
|
t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// `command` is the generic verb: the command line as given, split as a shell would, nothing added —
|
||||||
|
// so an operator's `node account g14 jochen` is one call through the console rather than a shell on
|
||||||
|
// the control node (novox/hq ADR 0154, ADR 0175).
|
||||||
|
func TestCommandRunsTheLineAsGiven(t *testing.T) {
|
||||||
|
argv, err := argvFor("command", map[string]any{"command": "node account g14 jochen"})
|
||||||
|
if err != nil || strings.Join(argv, " ") != "node account g14 jochen" {
|
||||||
|
t.Fatalf("a plain line: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("command", map[string]any{"command": `settings set dnsmasq '{"a": "b c"}' --node ace`})
|
||||||
|
if err != nil || len(argv) != 6 || argv[3] != `{"a": "b c"}` {
|
||||||
|
t.Fatalf("a quoted word stays one word: %q %v", argv, err)
|
||||||
|
}
|
||||||
|
argv, err = argvFor("command", map[string]any{"command": `node add "the box" --adopted`})
|
||||||
|
if err != nil || len(argv) != 4 || argv[2] != "the box" {
|
||||||
|
t.Fatalf("double quotes group: %q %v", argv, err)
|
||||||
|
}
|
||||||
|
if _, err := argvFor("command", map[string]any{"command": " "}); err == nil {
|
||||||
|
t.Fatal("an empty line was accepted")
|
||||||
|
}
|
||||||
|
if _, err := argvFor("command", map[string]any{"command": `node "unclosed`}); err == nil {
|
||||||
|
t.Fatal("an unclosed quote was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A verb in the row that this binary cannot run does not take the control plane off the bus: the
|
||||||
|
// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
|
||||||
|
// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
|
||||||
|
// a person running the binary by hand — the push that would have repaired it needs the control
|
||||||
|
// plane that was down.
|
||||||
|
func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
|
||||||
|
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||||
|
if !known {
|
||||||
|
t.Fatal("no controller seat")
|
||||||
|
}
|
||||||
|
// The row as a newer control plane would have written it: every verb this build knows, and one
|
||||||
|
// it does not.
|
||||||
|
widened := seat
|
||||||
|
widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
|
||||||
|
catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
|
||||||
|
rows := catalogue.DefaultSeats()
|
||||||
|
for i := range rows {
|
||||||
|
if rows[i].Name == catalogue.ControllerSeatName {
|
||||||
|
rows[i] = widened
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catalogue.UseSeats(rows)
|
||||||
|
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||||
|
|
||||||
|
handlers, behind, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
|
||||||
|
}
|
||||||
|
if len(behind) != 1 || behind[0] != "teleport" {
|
||||||
|
t.Fatalf("the verbs this build cannot run were reported as %v", behind)
|
||||||
|
}
|
||||||
|
if len(handlers) != len(widened.Serves) {
|
||||||
|
t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
|
||||||
|
}
|
||||||
|
for _, known := range []string{"status", "nodes", "push"} {
|
||||||
|
if handlers[known] == nil {
|
||||||
|
t.Errorf("%s is not served although this build knows it", known)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = handlers["teleport"](context.Background(), nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("the unknown verb answered as though it had run")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"teleport", "cannot run it", "behind"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Errorf("the answer does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON,
|
||||||
|
// as status and seats do, so nothing parses a printed column.
|
||||||
|
func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) {
|
||||||
|
for verb, want := range map[string]string{"nodes": "[node list --json]", "modules": "[module list --json]"} {
|
||||||
|
argv, err := argvFor(verb, map[string]any{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if fmt.Sprint(argv) != want {
|
||||||
|
t.Errorf("%s runs %v, want %s", verb, argv, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and
|
||||||
|
// queue it serves it on, with the seat's own description and schema, in NATS's services format.
|
||||||
|
func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) {
|
||||||
|
handlers, _, err := seatToolHandlers()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info := seatAnnouncement(handlers)
|
||||||
|
if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" {
|
||||||
|
t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity)
|
||||||
|
}
|
||||||
|
if len(info.Endpoints) != len(handlers) {
|
||||||
|
t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers))
|
||||||
|
}
|
||||||
|
for _, e := range info.Endpoints {
|
||||||
|
verb := e.Metadata["tool"]
|
||||||
|
if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb {
|
||||||
|
t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb)
|
||||||
|
}
|
||||||
|
if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName {
|
||||||
|
t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata)
|
||||||
|
}
|
||||||
|
if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName {
|
||||||
|
t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup)
|
||||||
|
}
|
||||||
|
if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" {
|
||||||
|
t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,421 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/secrets"
|
||||||
|
)
|
||||||
|
|
||||||
|
// secretCommand gives the mesh a value it must carry and could not have invented.
|
||||||
|
//
|
||||||
|
// **Every other secret in this mesh is one the mesh made** — generated, sealed to the machine that
|
||||||
|
// will use it, and never readable again. That is right for something coming into existence, and
|
||||||
|
// wrong for something that already exists: a database created last year has the password it was
|
||||||
|
// created with, and generating a new one puts 32 random bytes where a working credential was.
|
||||||
|
// The machine applies it, reports success, and whatever reads it fails to authenticate somewhere
|
||||||
|
// else entirely — with the mesh insisting the secret was delivered, which it was.
|
||||||
|
//
|
||||||
|
// So this is the entry point for **adopting** something already running. The store has carried
|
||||||
|
// the distinction since the beginning: a module secret records whether it was `made` or
|
||||||
|
// `accepted`, and refuses to invent a replacement for the second. Nothing until now could write
|
||||||
|
// one, so the only accepted secret in the mesh was the broker account issued to a build machine.
|
||||||
|
//
|
||||||
|
// The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
|
||||||
|
// **The only difference between the two is where the value came from.**
|
||||||
|
func secretCommand(ctx context.Context, args []string) error {
|
||||||
|
if len(args) == 0 {
|
||||||
|
return errors.New(secretUsage)
|
||||||
|
}
|
||||||
|
switch args[0] {
|
||||||
|
case "accept":
|
||||||
|
case "rotate":
|
||||||
|
return secretRotate(ctx, args[1:])
|
||||||
|
case "recover":
|
||||||
|
return secretRecover(ctx, args[1:])
|
||||||
|
case "export":
|
||||||
|
return secretExport(ctx, args[1:])
|
||||||
|
default:
|
||||||
|
return errors.New(secretUsage)
|
||||||
|
}
|
||||||
|
rest, flags := split(args[1:])
|
||||||
|
set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
|
||||||
|
from := set.String("from", "",
|
||||||
|
"read the value from this file instead of asking (use - for standard input)")
|
||||||
|
provider := set.String("provider", "",
|
||||||
|
"the node providing <name>: the value becomes the PAIR credential between <module> on <node> "+
|
||||||
|
"and that provider, sealed to both — the vault's operator-delivered secret (ADR 0092)")
|
||||||
|
local := set.String("local", "",
|
||||||
|
"with --provider: the name the credential goes by inside <module>, where its manifest keeps "+
|
||||||
|
"several for <name> (ADR 0094)")
|
||||||
|
if err := set.Parse(flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 3 {
|
||||||
|
return errors.New(secretUsage)
|
||||||
|
}
|
||||||
|
node, module, name := rest[0], rest[1], rest[2]
|
||||||
|
|
||||||
|
value, err := valueFor(node, module, name, *from)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
value = asSupplied(value)
|
||||||
|
if value == "" {
|
||||||
|
return errors.New("there is nothing to seal")
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
|
||||||
|
if *provider != "" {
|
||||||
|
// Into the pair, not into the module's own secrets: what the provider is asked to create
|
||||||
|
// and what the consumer reads are the same value, and neither end can be told a different
|
||||||
|
// one later without the other (novox/hq 04-ISSUES/070).
|
||||||
|
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local))
|
||||||
|
fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n")
|
||||||
|
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||||
|
// machine and the mesh cannot read it again.
|
||||||
|
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||||
|
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||||
|
fmt.Printf(" run `push %s` to send it\n", node)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const secretUsage = "secret rotate <node> <module> <name>\n" +
|
||||||
|
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
||||||
|
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||||
|
"secret export [--out <file>]"
|
||||||
|
|
||||||
|
// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
|
||||||
|
//
|
||||||
|
// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here
|
||||||
|
// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and
|
||||||
|
// no key for it, and this program holds the key for the length of the call and no blob until given
|
||||||
|
// one. Recovery needs both, which is what keeps the sealing meaningful.
|
||||||
|
//
|
||||||
|
// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the
|
||||||
|
// source mesh's secret tools were written after a secret was printed into a transcript, and that
|
||||||
|
// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery
|
||||||
|
// works with the store gone, which is the case it exists for.
|
||||||
|
func secretRecover(ctx context.Context, args []string) error {
|
||||||
|
rest, flags := split(args)
|
||||||
|
set := flag.NewFlagSet("secret recover", flag.ContinueOnError)
|
||||||
|
keyFile := set.String("key", "", "the operator's private key, from `operator key make`")
|
||||||
|
out := set.String("out", "", "where to write the value (0600); - for standard output. Default <node>.<module>.<name>.secret")
|
||||||
|
fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store")
|
||||||
|
provider := set.String("provider", "", "for a pair credential held from more than one provider: which one")
|
||||||
|
local := set.String("local", "", "for a pair credential the module keeps under a local name (ADR 0094): which one")
|
||||||
|
if err := set.Parse(flags); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(rest) != 3 {
|
||||||
|
return errors.New(secretUsage)
|
||||||
|
}
|
||||||
|
node, module, name := rest[0], rest[1], rest[2]
|
||||||
|
private, err := readPrivateKey(*keyFile)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
var kept inventory.Kept
|
||||||
|
if *fromExport != "" {
|
||||||
|
kept, err = keptFromExport(*fromExport, node, module, name, *provider)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
kept, err = open.inventory.KeptSecret(ctx, node, module, name, *provider, *local)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
value, err := secrets.Open(private, kept.Sealed)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w",
|
||||||
|
module, node, name, secrets.Fingerprint(kept.Key), err)
|
||||||
|
}
|
||||||
|
if *out == "-" {
|
||||||
|
_, err := os.Stdout.Write(append(value, '\n'))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
path := *out
|
||||||
|
if path == "" {
|
||||||
|
path = node + "." + module + "." + name + ".secret"
|
||||||
|
}
|
||||||
|
if err := writeNew(path, value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n",
|
||||||
|
module, node, name, path, len(value), kept.Origin)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault
|
||||||
|
// keeps the same document on its disk (Manifest.Keeps).
|
||||||
|
type export = catalogue.KeptExport
|
||||||
|
|
||||||
|
func secretExport(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("secret export", flag.ContinueOnError)
|
||||||
|
out := set.String("out", "", "where to write the export (0600); - or empty for standard output")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
key, err := inv.OperatorKey(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if key == "" {
|
||||||
|
return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first")
|
||||||
|
}
|
||||||
|
doc, err := inv.OperatorExport(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body, err := json.MarshalIndent(doc, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body = append(body, '\n')
|
||||||
|
if *out == "" || *out == "-" {
|
||||||
|
_, err := os.Stdout.Write(body)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Replaced whole, and made 0600 whether or not it existed: an export is ciphertext and a public
|
||||||
|
// key, but it is also the list of every secret the mesh has, and a file left at an earlier mode
|
||||||
|
// while the command says 0600 is a lie in the one place a person checks.
|
||||||
|
if err := writeReplacing(*out, body); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n",
|
||||||
|
len(doc.Kept), *out, doc.Fingerprint)
|
||||||
|
if len(doc.EarlierKey) > 0 {
|
||||||
|
fmt.Printf(" %d secret(s) are sealed to an EARLIER operator key: recoverable with that key only\n", len(doc.EarlierKey))
|
||||||
|
}
|
||||||
|
if len(doc.Unrecoverable) > 0 {
|
||||||
|
fmt.Printf(" %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(doc.Unrecoverable))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeNew writes a file that must not exist yet, atomically: create-exclusive, 0600. A check
|
||||||
|
// followed by a write is a window in which a key somebody still needs can be overwritten.
|
||||||
|
func writeNew(path string, content []byte) error {
|
||||||
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsExist(err) {
|
||||||
|
return fmt.Errorf("%s already exists; not overwriting it", path)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := f.Write(content); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeReplacing writes a file whole, creating or truncating it, and leaves it at 0600 either way.
|
||||||
|
func writeReplacing(path string, content []byte) error {
|
||||||
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := f.Write(content); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := f.Chmod(0o600); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func keptFromExport(path, node, module, name, provider string) (inventory.Kept, error) {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return inventory.Kept{}, err
|
||||||
|
}
|
||||||
|
var e export
|
||||||
|
if err := json.Unmarshal(raw, &e); err != nil {
|
||||||
|
return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err)
|
||||||
|
}
|
||||||
|
// Sealed to the current key or to an earlier one: both are copies the given key might open,
|
||||||
|
// and Open says which. Not the unrecoverable list, which holds no copy at all.
|
||||||
|
var found []inventory.Kept
|
||||||
|
for _, k := range append(append([]inventory.Kept{}, e.Kept...), e.EarlierKey...) {
|
||||||
|
if k.Node == node && k.Module == module && k.Name == name && (provider == "" || k.Provider == provider) {
|
||||||
|
found = append(found, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(found) {
|
||||||
|
case 0:
|
||||||
|
return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node)
|
||||||
|
case 1:
|
||||||
|
return found[0], nil
|
||||||
|
default:
|
||||||
|
providers := make([]string, 0, len(found))
|
||||||
|
for _, f := range found {
|
||||||
|
providers = append(providers, f.Provider)
|
||||||
|
}
|
||||||
|
return inventory.Kept{}, fmt.Errorf("%s holds %s's %q on %s from more than one provider (%s); say which with --provider",
|
||||||
|
path, module, name, node, strings.Join(providers, ", "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// split separates what this command is about from how it was asked.
|
||||||
|
//
|
||||||
|
// **Because the standard library stops parsing at the first non-flag argument.** With the
|
||||||
|
// positionals first — which is the order that reads correctly — everything after them is left
|
||||||
|
// sitting in the arguments, so `secret accept a b c --from -` arrives as five positionals and the
|
||||||
|
// flag is never seen. The host's own parser carries the same note, and the fault it names is
|
||||||
|
// worse than this one: there, a flag somebody passed was silently ignored and the command
|
||||||
|
// succeeded anyway.
|
||||||
|
func split(args []string) (positional, flags []string) {
|
||||||
|
for i, arg := range args {
|
||||||
|
if strings.HasPrefix(arg, "-") {
|
||||||
|
return args[:i], args[i:]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// asSupplied is the value with its line ending removed and nothing else.
|
||||||
|
//
|
||||||
|
// **A file has a trailing newline and a password does not**, so the ending goes — a credential
|
||||||
|
// wrong by one byte fails in a way nobody connects to how it was supplied.
|
||||||
|
//
|
||||||
|
// **And only the ending.** Trimming both ends is the obvious thing and it is wrong: a password
|
||||||
|
// chosen with a leading space is one the mesh would then deliver as a different password, silently,
|
||||||
|
// with the operator certain they had supplied it correctly.
|
||||||
|
func asSupplied(raw string) string {
|
||||||
|
return strings.TrimRight(raw, "\r\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// valueFor gets the secret without putting it somewhere it can be read afterwards.
|
||||||
|
//
|
||||||
|
// **Not an argument, and there is no flag that takes one.** A value on the command line is in the
|
||||||
|
// shell's history, in the process list for as long as it runs, and in whatever collects either.
|
||||||
|
// The paths here are a file the operator already has, or a prompt that does not echo — the same
|
||||||
|
// two ways a model-access key is supplied (novox/hq ADR 0024).
|
||||||
|
func valueFor(node, module, name, from string) (string, error) {
|
||||||
|
switch {
|
||||||
|
case from == "-":
|
||||||
|
body, err := io.ReadAll(os.Stdin)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(body), nil
|
||||||
|
|
||||||
|
case from != "":
|
||||||
|
body, err := os.ReadFile(from)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(body), nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
// The same path a model-access key takes, and for the same reason: a value given as an
|
||||||
|
// argument is in the shell's history and in the process list. Read from standard input,
|
||||||
|
// echoed nowhere by this program.
|
||||||
|
fmt.Fprintf(os.Stderr,
|
||||||
|
"reading %s's %q for %s from standard input; it is not echoed anywhere\n",
|
||||||
|
module, name, node)
|
||||||
|
line, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||||
|
if err != nil && line == "" {
|
||||||
|
return "", fmt.Errorf("nothing was given on standard input: %w", err)
|
||||||
|
}
|
||||||
|
return line, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
|
||||||
|
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
|
||||||
|
// this is the secret with one party. Said in the log with who asked and when, never the value.
|
||||||
|
func secretRotate(ctx context.Context, args []string) error {
|
||||||
|
rest, _ := split(args)
|
||||||
|
if len(rest) != 3 {
|
||||||
|
return errors.New(secretUsage)
|
||||||
|
}
|
||||||
|
node, module, name := rest[0], rest[1], rest[2]
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
|
||||||
|
var refused inventory.ErrNotRotatable
|
||||||
|
if errors.As(err, &refused) {
|
||||||
|
return fmt.Errorf("not rotated: %s", refused.Why)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
|
||||||
|
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
|
||||||
|
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
|
||||||
|
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
|
||||||
|
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(machines) == 0 {
|
||||||
|
machines = []string{node}
|
||||||
|
}
|
||||||
|
if len(machines) == 1 {
|
||||||
|
fmt.Printf("sending %s, so %s starts again on the new value:\n", node, module)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("shared with every consumer; sending %s together:\n", strings.Join(machines, ", "))
|
||||||
|
}
|
||||||
|
if err := sendTo(ctx, open, machines); err != nil {
|
||||||
|
return fmt.Errorf("%w\n\nThe new value is sealed and not yet delivered; what runs keeps the old "+
|
||||||
|
"one until the machines next apply. Fix the cause and run `push --behind`", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// whoAsked names the caller for the log: the account the command runs as, which for a tool call
|
||||||
|
// through the console is the mesh's own.
|
||||||
|
func whoAsked() string {
|
||||||
|
if u := os.Getenv("SUDO_USER"); u != "" {
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
if u := os.Getenv("USER"); u != "" {
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
return "the mesh"
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A file has a trailing newline and a password does not.
|
||||||
|
//
|
||||||
|
// The failure this prevents is the worst kind to diagnose: the credential is delivered, the
|
||||||
|
// machine applies it, everything reports success, and authentication fails one byte from correct
|
||||||
|
// somewhere else entirely.
|
||||||
|
func TestATrailingNewlineIsNotPartOfTheSecret(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "password")
|
||||||
|
if err := os.WriteFile(path, []byte("the-database-password\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := valueFor("anchor", "umami", "database", path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asSupplied(got) != "the-database-password" {
|
||||||
|
t.Fatalf("read %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A password may contain spaces, and they are the operator's.
|
||||||
|
//
|
||||||
|
// Trimming both ends is the obvious thing and it is wrong: a value chosen with a leading space is
|
||||||
|
// a value the mesh would silently deliver as a different one.
|
||||||
|
func TestOnlyLineEndingsAreRemoved(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "password")
|
||||||
|
if err := os.WriteFile(path, []byte(" spaces matter \n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := valueFor("anchor", "umami", "database", path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if asSupplied(got) != " spaces matter " {
|
||||||
|
t.Fatalf("the value was altered beyond its line ending: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file that is not there is said plainly, rather than becoming an empty secret.
|
||||||
|
func TestAMissingFileIsRefused(t *testing.T) {
|
||||||
|
if _, err := valueFor("anchor", "umami", "database",
|
||||||
|
filepath.Join(t.TempDir(), "absent")); err == nil {
|
||||||
|
t.Fatal("a missing file produced a value")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The command refuses what it cannot act on, rather than acting on part of it.
|
||||||
|
func TestTheArgumentsAreRequired(t *testing.T) {
|
||||||
|
for _, args := range [][]string{
|
||||||
|
{},
|
||||||
|
{"accept"},
|
||||||
|
{"accept", "anchor"},
|
||||||
|
{"accept", "anchor", "umami"},
|
||||||
|
{"give", "anchor", "umami", "database"},
|
||||||
|
} {
|
||||||
|
if err := secretCommand(t.Context(), args); err == nil {
|
||||||
|
t.Errorf("%v was accepted", args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The invocation that actually gets used, which the first version of these tests never tried.
|
||||||
|
//
|
||||||
|
// Every case here was a rejection, so the command was broken in the one way that matters — it
|
||||||
|
// refused what it is for — and the tests were green. The lab found it at the first call.
|
||||||
|
func TestTheArgumentsAndTheFlagAreBothSeen(t *testing.T) {
|
||||||
|
rest, flags := split([]string{"anchor", "umami", "database", "--from", "-"})
|
||||||
|
assert(t, len(rest) == 3, "the three positionals were not kept: %v", rest)
|
||||||
|
assert(t, len(flags) == 2, "the flag was not separated: %v", flags)
|
||||||
|
|
||||||
|
// And with no flag at all, which is the interactive form.
|
||||||
|
rest, flags = split([]string{"anchor", "umami", "database"})
|
||||||
|
assert(t, len(rest) == 3 && len(flags) == 0, "%v / %v", rest, flags)
|
||||||
|
|
||||||
|
// A flag before the positionals still works, because somebody will write it that way.
|
||||||
|
rest, flags = split([]string{"--from", "/tmp/x"})
|
||||||
|
assert(t, len(rest) == 0 && len(flags) == 2, "%v / %v", rest, flags)
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the wiring, not just the helper.
|
||||||
|
//
|
||||||
|
// Testing `split` alone left the command able to ignore it entirely — removing the call changed
|
||||||
|
// no test. This reaches secretCommand: with a `--from` naming a file that is not there, the
|
||||||
|
// complaint must be about the file. A complaint about usage would mean the flag was never seen.
|
||||||
|
func TestTheCommandItselfSeesTheFlag(t *testing.T) {
|
||||||
|
err := secretCommand(t.Context(),
|
||||||
|
[]string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a missing file was accepted")
|
||||||
|
}
|
||||||
|
if strings.Contains(err.Error(), "secret accept <node>") {
|
||||||
|
t.Fatalf("the command did not see its flag and complained about usage instead: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assert(t *testing.T, ok bool, format string, args ...any) {
|
||||||
|
t.Helper()
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf(format, args...)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// sendable is a declaration as a machine is sent it: its resources and, for an adopted node, its
|
||||||
|
// mode and which modules were taken on it (novox/hq ADR 0100).
|
||||||
|
//
|
||||||
|
// **Body is the only place the envelope is marshalled.** It was written by hand at every send site,
|
||||||
|
// in the digest the mesh compares, and in `plan --json`; a key added at one and not another would
|
||||||
|
// make a machine look out of date for ever, or send something `plan` never showed.
|
||||||
|
type sendable struct {
|
||||||
|
Resources []map[string]any
|
||||||
|
// Sequence orders this send against every other to the same node: one higher each time, taken
|
||||||
|
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
|
||||||
|
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
|
||||||
|
Sequence int64
|
||||||
|
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||||
|
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||||
|
Adoption *adoptionEnvelope
|
||||||
|
|
||||||
|
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
|
||||||
|
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
|
||||||
|
// the same composition as its received files, and every machine's private-network address.
|
||||||
|
Received map[string]map[string][]catalogue.Contribution
|
||||||
|
Mesh []string
|
||||||
|
// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
|
||||||
|
// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
|
||||||
|
// whether that machine must go first is read from the list alone (novox/hq issue 249).
|
||||||
|
BusUsers string
|
||||||
|
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
||||||
|
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
||||||
|
// keeps that module's held things and touches none of its containers; a machine is told
|
||||||
|
// everything or nothing about what it IS told, and what it is not told is said. Absent from
|
||||||
|
// the body when empty, so a declaration that leaves nothing out is byte for byte what it was.
|
||||||
|
LeftOut []string
|
||||||
|
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||||
|
leftOutWhy map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||||
|
// it that it runs; Untaken is, for every module it runs that is not taken, the ids of every one of
|
||||||
|
// that module's resources — what the host keeps as found until the module is taken (ADR 0103). Ids
|
||||||
|
// rather than a rule to split them by, because a module's name may contain a dot.
|
||||||
|
type adoptionEnvelope struct {
|
||||||
|
Taken []string `json:"taken"`
|
||||||
|
Untaken map[string][]string `json:"untaken,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Body is the declaration's bytes, as sent and as digested.
|
||||||
|
func (s sendable) Body() ([]byte, error) {
|
||||||
|
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||||
|
if s.Adoption != nil {
|
||||||
|
envelope["adoption"] = s.Adoption
|
||||||
|
}
|
||||||
|
if s.Sequence > 0 {
|
||||||
|
envelope["sequence"] = s.Sequence
|
||||||
|
}
|
||||||
|
if len(s.LeftOut) > 0 {
|
||||||
|
envelope["left_out"] = s.LeftOut
|
||||||
|
}
|
||||||
|
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||||
|
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||||
|
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||||
|
if len(s.Resources) == 0 {
|
||||||
|
envelope["owns_nothing"] = true
|
||||||
|
}
|
||||||
|
return json.Marshal(envelope)
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionOf is the envelope for a node, nil when it is converged.
|
||||||
|
func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node,
|
||||||
|
plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) {
|
||||||
|
if !record.Adopted {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
taken, err := inv.Taken(ctx, record.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return adoptionFor(plan, taken, composed), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// adoptionFor is the envelope computed from what was taken and who owns each resource.
|
||||||
|
//
|
||||||
|
// Every module the node runs that is not taken is untaken — including one pulled in by another
|
||||||
|
// rather than assigned: what is found is kept until its module is taken, whoever put it there.
|
||||||
|
func adoptionFor(plan catalogue.Resolution, taken []string,
|
||||||
|
composed catalogue.Composed) *adoptionEnvelope {
|
||||||
|
isTaken := map[string]bool{}
|
||||||
|
for _, m := range taken {
|
||||||
|
isTaken[m] = true
|
||||||
|
}
|
||||||
|
out := &adoptionEnvelope{Taken: []string{}}
|
||||||
|
runs := map[string]bool{}
|
||||||
|
for _, m := range plan.Modules {
|
||||||
|
runs[m.Module] = true
|
||||||
|
if isTaken[m.Module] {
|
||||||
|
out.Taken = append(out.Taken, m.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out.Taken)
|
||||||
|
for _, r := range composed.Resources {
|
||||||
|
id, _ := r["id"].(string)
|
||||||
|
module, owned := composed.Owner[id]
|
||||||
|
// Every kind, not only files and containers (novox/hq ADR 0103): a directory, a service, a
|
||||||
|
// container mounting what was found and an action run in a held container all reach what
|
||||||
|
// the machine already has. What the mesh declares of its own is never held.
|
||||||
|
if !owned || !runs[module] || isTaken[module] ||
|
||||||
|
strings.HasPrefix(id, catalogue.AdoptionPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if out.Untaken == nil {
|
||||||
|
out.Untaken = map[string][]string{}
|
||||||
|
}
|
||||||
|
out.Untaken[module] = append(out.Untaken[module], id)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,443 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules
|
||||||
|
// were taken on it; a converged node's declaration is byte for byte what it was.
|
||||||
|
|
||||||
|
// helloWeb is a module the predecessor also runs: a page and a server under names it uses.
|
||||||
|
func helloWeb() catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||||
|
Resources: []map[string]any{
|
||||||
|
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hello"},
|
||||||
|
{"id": "server", "type": "container", "name": "hello-web",
|
||||||
|
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||||
|
{"id": "served", "type": "directory", "path": "/var/lib/hello-web"},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// composed is what node would be sent now, as push composes it.
|
||||||
|
func composed(t *testing.T, open *stores, node string) sendable {
|
||||||
|
t.Helper()
|
||||||
|
plan, settings, err := planFor(t.Context(), open, node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return declared
|
||||||
|
}
|
||||||
|
|
||||||
|
// convergedBefore is the envelope a converged node was sent before adoption existed, captured by
|
||||||
|
// running this same composition at the commit this branch left main (0a39b7d). The mesh here is
|
||||||
|
// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
|
||||||
|
// what changes this string is a change to what a converged machine is sent, which is the thing an
|
||||||
|
// older host would refuse.
|
||||||
|
//
|
||||||
|
// Re-captured 2026-10-01 (novox/hq issue 177): c978aa7 took `hosts` off every container — a
|
||||||
|
// machine's own resolver knows the mesh's names now — and left this string carrying it, so the
|
||||||
|
// guard failed for a day and nothing ran it. A field an older host never sees is the one change
|
||||||
|
// this guard permits; a field it would refuse is the one it exists to catch.
|
||||||
|
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
|
||||||
|
|
||||||
|
func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, helloWeb())
|
||||||
|
if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared := composed(t, open, "laptop")
|
||||||
|
if declared.Adoption != nil {
|
||||||
|
t.Fatal("a converged node was given an adoption envelope")
|
||||||
|
}
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(body) != convergedBefore {
|
||||||
|
t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s",
|
||||||
|
body, convergedBefore)
|
||||||
|
}
|
||||||
|
if bytes.Contains(body, []byte(`"adoption"`)) {
|
||||||
|
t.Fatal("a converged declaration names adoption; an older host would refuse it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, helloWeb())
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
declared := composed(t, open, "anchor")
|
||||||
|
if declared.Adoption == nil {
|
||||||
|
t.Fatal("an adopted node's declaration does not say it is adopted")
|
||||||
|
}
|
||||||
|
untaken := declared.Adoption.Untaken["hello-web"]
|
||||||
|
// Every kind — its directory too (novox/hq ADR 0103).
|
||||||
|
if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server",
|
||||||
|
"hello-web.served"}) {
|
||||||
|
t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if len(declared.Adoption.Taken) != 0 {
|
||||||
|
t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var envelope map[string]any
|
||||||
|
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
adoption, _ := envelope["adoption"].(map[string]any)
|
||||||
|
if _, ok := adoption["taken"].([]any); !ok {
|
||||||
|
t.Fatalf("taken is not a list on the wire, even empty: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The digest the mesh compares is the digest of what is sent: status and push agree.
|
||||||
|
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if would["anchor"] != digestOf(body) {
|
||||||
|
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||||
|
}
|
||||||
|
|
||||||
|
// plan --json prints that same envelope.
|
||||||
|
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||||
|
var compact bytes.Buffer
|
||||||
|
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||||
|
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||||
|
}
|
||||||
|
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||||
|
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Taking the module moves its resources out of untaken.
|
||||||
|
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
declared = composed(t, open, "anchor")
|
||||||
|
if _, still := declared.Adoption.Untaken["hello-web"]; still {
|
||||||
|
t.Fatalf("a taken module is still untaken: %v", declared.Adoption)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) {
|
||||||
|
t.Fatalf("taken is %v", declared.Adoption.Taken)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustNodes(t *testing.T, open *stores) []inventory.Node {
|
||||||
|
t.Helper()
|
||||||
|
nodes, err := open.inventory.Nodes(t.Context())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return nodes
|
||||||
|
}
|
||||||
|
|
||||||
|
// stdoutOf is what run printed.
|
||||||
|
func stdoutOf(t *testing.T, run func() error) string {
|
||||||
|
t.Helper()
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
saved := os.Stdout
|
||||||
|
os.Stdout = w
|
||||||
|
done := make(chan string)
|
||||||
|
go func() {
|
||||||
|
all, _ := io.ReadAll(r)
|
||||||
|
done <- string(all)
|
||||||
|
}()
|
||||||
|
runErr := run()
|
||||||
|
os.Stdout = saved
|
||||||
|
w.Close()
|
||||||
|
out := <-done
|
||||||
|
if runErr != nil {
|
||||||
|
t.Fatalf("%v\n%s", runErr, out)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other
|
||||||
|
// machines are told to reach it, and a port given for the whole mesh is refused.
|
||||||
|
func TestConsumersAreToldTheGivenPort(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||||
|
Guards: []int{5432},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"},
|
||||||
|
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}})
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "store",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
plan, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var told any
|
||||||
|
for _, n := range plan.Needs {
|
||||||
|
if n.Name == "database" {
|
||||||
|
told = n.Serves["port"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if told != 5433 {
|
||||||
|
t.Fatalf("the consumer is told the store is on %v", told)
|
||||||
|
}
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) {
|
||||||
|
t.Fatalf("the store publishes %v", r["ports"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A port for the whole mesh is refused where it is set, not stored to refuse every node's
|
||||||
|
// declaration afterwards.
|
||||||
|
if err := open.inventory.SetSettings(ctx, "", "store",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "per node") {
|
||||||
|
t.Fatalf("a port given for the whole mesh was not refused: %v", err)
|
||||||
|
}
|
||||||
|
plan, settings, err := planFor(ctx, open, "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil {
|
||||||
|
t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store
|
||||||
|
// assigned but not taken is not guarded — its port may still be the predecessor's — and taking it
|
||||||
|
// guards it from the next declaration.
|
||||||
|
func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||||
|
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||||
|
Guards: []int{5432},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||||
|
"ports": []any{"5432:5432"},
|
||||||
|
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) {
|
||||||
|
t.Fatal("an untaken store is guarded")
|
||||||
|
}
|
||||||
|
if err := open.inventory.Take(ctx, "anchor", "store"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, r := range composed(t, open, "anchor").Resources {
|
||||||
|
if r["id"] == catalogue.GuardID() {
|
||||||
|
if r["content"] != catalogue.AsGuard([]int{5432}) {
|
||||||
|
t.Fatalf("the taken store's guard is:\n%s", r["content"])
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatal("a taken store is not guarded")
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq ADR 0038 and 0100: a module may publish a port the long way — `2222:22`, because the
|
||||||
|
// machine's own ssh daemon holds 22 — and say it listens on the machine side of that mapping,
|
||||||
|
// which is the number anything reaching it dials. A node moves that port by naming it, and the
|
||||||
|
// number has to reach everything derived from it at once: what the runtime is handed, what an
|
||||||
|
// adopted node is told to open, what its guard refuses, and what a consumer elsewhere dials.
|
||||||
|
//
|
||||||
|
// It reached none of them. The setting was refused outright for naming the machine side — so a
|
||||||
|
// module's port could not be put back where the machine it replaces had it, and, worse, the
|
||||||
|
// node's every push failed for as long as the setting existed.
|
||||||
|
func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, catalogue.Manifest{Module: "forge", Version: "1",
|
||||||
|
Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}},
|
||||||
|
Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}},
|
||||||
|
Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}},
|
||||||
|
Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge",
|
||||||
|
"ports": []any{"2222:22"},
|
||||||
|
"image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}})
|
||||||
|
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||||
|
Requires: []string{"git-over-ssh"}})
|
||||||
|
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "anchor", "forge"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "anchor", "forge",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
resources := composed(t, open, "anchor").Resources
|
||||||
|
var container, opening map[string]any
|
||||||
|
for _, r := range resources {
|
||||||
|
switch r["id"] {
|
||||||
|
case "forge.server":
|
||||||
|
container = r
|
||||||
|
case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded):
|
||||||
|
opening = r
|
||||||
|
}
|
||||||
|
if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") {
|
||||||
|
t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) {
|
||||||
|
t.Fatalf("the forge's container publishes %v", container["ports"])
|
||||||
|
}
|
||||||
|
if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh {
|
||||||
|
t.Fatalf("no opening for the port this node gave the forge: %v", opening)
|
||||||
|
}
|
||||||
|
var guard map[string]any
|
||||||
|
for _, r := range resources {
|
||||||
|
if r["id"] == catalogue.GuardID() {
|
||||||
|
guard = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) {
|
||||||
|
t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the consumer on the other machine dials the same number.
|
||||||
|
plan, _, err := planFor(ctx, open, "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var told any
|
||||||
|
for _, n := range plan.Needs {
|
||||||
|
if n.Name == "git-over-ssh" {
|
||||||
|
told = n.Serves["port"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if told != 222 {
|
||||||
|
t.Fatalf("the consumer is told the forge answers on %v", told)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
|
||||||
|
// The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127).
|
||||||
|
body, err := sendable{}.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if env["owns_nothing"] != true {
|
||||||
|
t.Fatalf("an empty declaration must mark owns_nothing; got %v", env)
|
||||||
|
}
|
||||||
|
// A declaration with resources does not carry the marker.
|
||||||
|
body, _ = sendable{Resources: []map[string]any{{"id": "x"}}}.Body()
|
||||||
|
var env2 map[string]any
|
||||||
|
_ = json.Unmarshal(body, &env2)
|
||||||
|
if _, present := env2["owns_nothing"]; present {
|
||||||
|
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A setting is judged where it is stored, and an impossible one costs a module, not a machine
|
||||||
|
// (novox/hq ADR 0163, rule 6): stored while it composed, a setting whose definition then moved from
|
||||||
|
// under it leaves that module out of the declaration — said in the envelope, so the host keeps the
|
||||||
|
// module's things — and the machine is told everything else.
|
||||||
|
func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
web := helloWeb()
|
||||||
|
web.Resources[1]["ports"] = []any{"8080"}
|
||||||
|
register(t, open, web)
|
||||||
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}})
|
||||||
|
for _, m := range []string{"hello-web", "notes"} {
|
||||||
|
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Judged where it is stored: a port the module does not publish is refused by name.
|
||||||
|
err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") {
|
||||||
|
t.Fatalf("an impossible setting was stored: %v", err)
|
||||||
|
}
|
||||||
|
if err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
|
||||||
|
map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 {
|
||||||
|
t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The definition moves: the container publishes another port now.
|
||||||
|
web.Version = "2"
|
||||||
|
web.Resources[1]["ports"] = []any{"9090"}
|
||||||
|
register(t, open, web)
|
||||||
|
declared := composed(t, open, "laptop")
|
||||||
|
if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" {
|
||||||
|
t.Fatalf("hello-web is not left out: %v", declared.LeftOut)
|
||||||
|
}
|
||||||
|
if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") {
|
||||||
|
t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy)
|
||||||
|
}
|
||||||
|
if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") {
|
||||||
|
t.Fatalf("the machine was not told everything else: %v", declared.Resources)
|
||||||
|
}
|
||||||
|
body, err := declared.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, _ := env["left_out"].([]any)
|
||||||
|
if len(left) != 1 || left[0] != "hello-web" {
|
||||||
|
t.Fatalf("the envelope does not say what was left out: %v", env)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A declaration's only identity was the digest of its bytes; the controller already held a per-node
|
||||||
|
// lock and recorded each send, so the order existed and was thrown away at the wire (novox/hq
|
||||||
|
// 04-ISSUES/107).
|
||||||
|
|
||||||
|
func TestASendCarriesItsNumberInsideTheSignedBytes(t *testing.T) {
|
||||||
|
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}, Sequence: 7}.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := env["sequence"].(float64); got != 7 {
|
||||||
|
t.Fatalf("the body carries sequence %v, wanted 7", env["sequence"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) {
|
||||||
|
// Zero is not sent at all. A host reads absence as "no order claimed" — the shape of every
|
||||||
|
// declaration before this — so an older host, or the read-only comparison against a machine
|
||||||
|
// sent nothing since sends were numbered, sees exactly the bytes it always saw.
|
||||||
|
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}}.Body()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var env map[string]any
|
||||||
|
if err := json.Unmarshal(body, &env); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, present := env["sequence"]; present {
|
||||||
|
t.Fatalf("a send numbered zero put a sequence on the wire: %s", body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Sent nothing since numbering existed: what it would be sent is composed with zero, which is
|
||||||
|
// not on the wire, which is what it was actually sent.
|
||||||
|
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 {
|
||||||
|
t.Fatalf("a fresh node reads sequence %d, %v", n, err)
|
||||||
|
}
|
||||||
|
first, err := open.inventory.NextSequence(t.Context(), record.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := open.inventory.NextSequence(t.Context(), record.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if first != 1 || second != 2 {
|
||||||
|
t.Fatalf("two sends were numbered %d and %d", first, second)
|
||||||
|
}
|
||||||
|
// And the read path sees the last one taken, so the comparison composes what was sent.
|
||||||
|
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 {
|
||||||
|
t.Fatalf("after two sends the node reads sequence %d, %v", n, err)
|
||||||
|
}
|
||||||
|
// Another node counts on its own.
|
||||||
|
other, err := open.inventory.NodeByName(t.Context(), "laptop")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 {
|
||||||
|
t.Fatalf("a second node's first send was numbered %d, %v", n, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// where a build's repository is (novox/hq ADR 0111).
|
||||||
|
//
|
||||||
|
// A repository is on the mesh's own forge, or it is anywhere else. The first is recorded as its path
|
||||||
|
// on the forge holding the git seat, and cloned from wherever that forge runs at the moment of
|
||||||
|
// building; the second is a URL, recorded and cloned exactly as given. The build machine is not told
|
||||||
|
// the difference — it is handed a URL either way — because only the control plane knows where the
|
||||||
|
// seat's holder runs.
|
||||||
|
|
||||||
|
// gitSeat is the seat a self-hosted repository lives on.
|
||||||
|
const gitSeat = "git"
|
||||||
|
|
||||||
|
// buildSource is where a build's repository is: a URL, or a path on a seat's holder.
|
||||||
|
type buildSource struct {
|
||||||
|
Repository string
|
||||||
|
Seat string
|
||||||
|
}
|
||||||
|
|
||||||
|
// String is the source as a person reads it, which for one on a seat is not the URL: the URL is a
|
||||||
|
// fact about where the forge happens to run today.
|
||||||
|
func (s buildSource) String() string {
|
||||||
|
if s.Seat == "" {
|
||||||
|
return s.Repository
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s on the %s seat", s.Repository, s.Seat)
|
||||||
|
}
|
||||||
|
|
||||||
|
// onASeat refuses an address given as a path on the forge.
|
||||||
|
//
|
||||||
|
// **A URL here would be recorded as a path**, and then composed onto the forge's address as one —
|
||||||
|
// cloning `http://forge:3000/https://github.com/…`. Refused by what an address plainly looks like,
|
||||||
|
// not repaired: `--self` promises a path, and something that is not one is a mistake to name.
|
||||||
|
func onASeat(repository string) error {
|
||||||
|
if strings.Contains(repository, ":") || strings.HasPrefix(repository, "/") ||
|
||||||
|
strings.Trim(repository, "/") == "" {
|
||||||
|
return fmt.Errorf("--self takes the repository's path on the forge, such as novox/mesh-catalog, "+
|
||||||
|
"and %q is not one — without --self it is built from exactly what is given", repository)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cloneFrom is the URL a build machine clones for a source.
|
||||||
|
//
|
||||||
|
// A URL is itself. A path on a seat is composed from the seat's holder as the mesh sees it now —
|
||||||
|
// the same view planning takes of every machine, so the forge a build clones from is the forge the
|
||||||
|
// mesh says holds the seat.
|
||||||
|
func cloneFrom(ctx context.Context, source buildSource) (string, error) {
|
||||||
|
if source.Seat == "" {
|
||||||
|
return source.Repository, nil
|
||||||
|
}
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return clonedFromSeat(world, source.Seat, source.Repository)
|
||||||
|
}
|
||||||
|
|
||||||
|
// clonedFromSeat composes the clone URL for a repository on a seat's holder.
|
||||||
|
//
|
||||||
|
// **Refused, never defaulted, at every step that has no answer.** Nobody holding the seat is a mesh
|
||||||
|
// without a forge of its own: it builds from external repositories and must say so rather than fail
|
||||||
|
// to clone. A holder off the private network cannot be reached by any build machine. A holder that
|
||||||
|
// serves no scheme or port has nothing to compose from — a default port here would be the forge's
|
||||||
|
// address guessed, which is the thing this exists to stop.
|
||||||
|
func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) {
|
||||||
|
base, err := seatBase(world, seatName)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
|
||||||
|
return fmt.Sprintf("%s/%s.git", base, path), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatBase is `scheme://host:port` of a seat's holder as the mesh reaches it, for cloning.
|
||||||
|
func seatBase(world catalogue.World, seatName string) (string, error) {
|
||||||
|
seat, known := catalogue.SeatNamed(seatName)
|
||||||
|
if !known || seat.Delivers == "" {
|
||||||
|
return "", fmt.Errorf("%q is not a seat a repository can live on", seatName)
|
||||||
|
}
|
||||||
|
var holder *catalogue.Held
|
||||||
|
for i, h := range world.Held {
|
||||||
|
if h.Claim == seat.Name && h.Scope == seat.Scope {
|
||||||
|
holder = &world.Held[i]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if holder == nil {
|
||||||
|
return "", fmt.Errorf("nobody holds the %s seat, so nothing can be cloned from this mesh's "+
|
||||||
|
"forge — assign a module that claims it, or build from the repository's URL without --self",
|
||||||
|
seat.Name)
|
||||||
|
}
|
||||||
|
var provider *catalogue.Provider
|
||||||
|
for i, p := range world.Offered[seat.Delivers] {
|
||||||
|
if p.Node == holder.Node && p.Module == holder.Module {
|
||||||
|
provider = &world.Offered[seat.Delivers][i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if provider == nil {
|
||||||
|
return "", fmt.Errorf("%s on %s holds the %s seat and offers no %q to clone from",
|
||||||
|
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||||
|
}
|
||||||
|
if provider.At == "" {
|
||||||
|
return "", fmt.Errorf("%s on %s holds the %s seat and is not on the private network, so no "+
|
||||||
|
"build machine can reach it", holder.Module, holder.Node, seat.Name)
|
||||||
|
}
|
||||||
|
scheme, _ := provider.Serves["scheme"].(string)
|
||||||
|
port := servedPort(provider.Serves["port"])
|
||||||
|
if scheme == "" || port == "" {
|
||||||
|
return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q",
|
||||||
|
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s://%s:%s", scheme, provider.At, port), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// seatBases is the clone base of every seat a recipe's context may name, for a build request
|
||||||
|
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
|
||||||
|
// name it at all, and if it does the builder refuses with the seat's name.
|
||||||
|
func seatBases(ctx context.Context) map[string]string {
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
bases := map[string]string{}
|
||||||
|
for _, seatName := range []string{gitSeat} {
|
||||||
|
if base, err := seatBase(world, seatName); err == nil {
|
||||||
|
bases[seatName] = base
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return bases
|
||||||
|
}
|
||||||
|
|
||||||
|
// servedPort is a served port as text, however the manifest and the node's settings carried it.
|
||||||
|
func servedPort(v any) string {
|
||||||
|
switch p := v.(type) {
|
||||||
|
case float64:
|
||||||
|
return strconv.Itoa(int(p))
|
||||||
|
case int:
|
||||||
|
return strconv.Itoa(p)
|
||||||
|
case string:
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Defends novox/hq ADR 0111: a build source is on the git seat, or it is external.
|
||||||
|
|
||||||
|
func forgeHolding(port any) catalogue.World {
|
||||||
|
return catalogue.World{
|
||||||
|
Held: []catalogue.Held{{Claim: "git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}},
|
||||||
|
Offered: map[string][]catalogue.Provider{"git": {
|
||||||
|
// A second forge that does not hold the seat, so taking the first one found would be wrong.
|
||||||
|
{Node: "archive", At: "archive.internal", Module: "gitea-mirror",
|
||||||
|
Serves: map[string]any{"scheme": "http", "port": float64(3000)}},
|
||||||
|
{Node: "anchor", At: "anchor.internal", Module: "gitea",
|
||||||
|
Serves: map[string]any{"scheme": "http", "port": port}},
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) {
|
||||||
|
got, err := clonedFromSeat(forgeHolding(float64(3000)), "git", "novox/mesh-catalog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != "http://anchor.internal:3000/novox/mesh-catalog.git" {
|
||||||
|
t.Fatalf("cloned from %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) {
|
||||||
|
// The whole point: the node gave the forge another port, and the same recorded path clones
|
||||||
|
// from the new one. Nothing recorded contained the old one to be wrong.
|
||||||
|
got, err := clonedFromSeat(forgeHolding(float64(3100)), "git", "novox/mesh-catalog")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, ":3100/") {
|
||||||
|
t.Fatalf("the moved port was not followed: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithNobodyHoldingTheSeatASelfHostedBuildIsRefusedAndSaysWhy(t *testing.T) {
|
||||||
|
_, err := clonedFromSeat(catalogue.World{}, "git", "novox/mesh-catalog")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a repository was cloned from a forge the mesh does not have")
|
||||||
|
}
|
||||||
|
for _, want := range []string{"nobody holds the git seat", "without --self"} {
|
||||||
|
if !strings.Contains(err.Error(), want) {
|
||||||
|
t.Fatalf("the refusal does not say %q: %v", want, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnExternalRepositoryIsClonedExactlyAsGiven(t *testing.T) {
|
||||||
|
// Unaffected by the seat, held or not: GitHub and GitLab are the ordinary cases.
|
||||||
|
given := "https://github.com/someone/something.git"
|
||||||
|
got, err := cloneFrom(t.Context(), buildSource{Repository: given})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != given {
|
||||||
|
t.Fatalf("an external repository became %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) {
|
||||||
|
world := forgeHolding(float64(3000))
|
||||||
|
world.Offered["git"][1].At = ""
|
||||||
|
if _, err := clonedFromSeat(world, "git", "novox/mesh-catalog"); err == nil ||
|
||||||
|
!strings.Contains(err.Error(), "private network") {
|
||||||
|
t.Fatalf("a forge nothing can reach was cloned from: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHolderServingNoPortIsRefusedRatherThanGuessed(t *testing.T) {
|
||||||
|
// A default port would be the forge's address guessed, which is what this exists to stop.
|
||||||
|
if _, err := clonedFromSeat(forgeHolding(nil), "git", "novox/mesh-catalog"); err == nil {
|
||||||
|
t.Fatal("a port was guessed for a forge that serves none")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAddressGivenAsAPathOnTheForgeIsRefused(t *testing.T) {
|
||||||
|
for _, bad := range []string{
|
||||||
|
"https://github.com/someone/something.git",
|
||||||
|
"git@anchor:novox/mesh-catalog.git",
|
||||||
|
"/srv/git/mesh-catalog",
|
||||||
|
"",
|
||||||
|
} {
|
||||||
|
if err := onASeat(bad); err == nil {
|
||||||
|
t.Errorf("--self accepted %q as a path on the forge", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := onASeat("novox/mesh-catalog"); err != nil {
|
||||||
|
t.Errorf("a path on the forge was refused: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestASourceOnTheSeatReadsAsAPathNotAnAddress(t *testing.T) {
|
||||||
|
s := buildSource{Repository: "novox/mesh-catalog", Seat: "git"}
|
||||||
|
if got := s.String(); got != "novox/mesh-catalog on the git seat" {
|
||||||
|
t.Fatalf("read as %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,562 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/overlay"
|
||||||
|
)
|
||||||
|
|
||||||
|
// is anything broken, is anything not answering, is anything out of date.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
// short is a commit as a person refers to it.
|
||||||
|
func short(commit string) string {
|
||||||
|
if len(commit) > 8 {
|
||||||
|
return commit[:8]
|
||||||
|
}
|
||||||
|
return commit
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusCommand answers "did my change go out?".
|
||||||
|
//
|
||||||
|
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
|
||||||
|
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
|
||||||
|
// this is worse to live with whatever its other properties.
|
||||||
|
//
|
||||||
|
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
|
||||||
|
// source now has, and which machines are running the old one.
|
||||||
|
func statusCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||||
|
asJSON := set.Bool("json", false, "the same answers, for something other than a person")
|
||||||
|
if _, err := parseAround(set, args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
return statusFor(ctx, open, *asJSON)
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusFor asks and answers, against stores somebody else opened.
|
||||||
|
//
|
||||||
|
// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
|
||||||
|
// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
|
||||||
|
// words the thing worth holding still.
|
||||||
|
func statusFor(ctx context.Context, open *stores, asJSON bool) error {
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if asJSON {
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(string(body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return printStatus(asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// printStatus is the words, separated from the questions.
|
||||||
|
//
|
||||||
|
// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
|
||||||
|
// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
|
||||||
|
// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
|
||||||
|
// test can read them without a store, a bus or a machine.
|
||||||
|
func printStatus(asked answers) error {
|
||||||
|
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||||
|
behind, sources := asked.behind, asked.sources
|
||||||
|
|
||||||
|
if len(asked.refused) > 0 {
|
||||||
|
// First, above everything else. A machine that cannot be worked out is not running an old
|
||||||
|
// declaration — it has no declaration, and nothing below this line is about it.
|
||||||
|
var names []string
|
||||||
|
for name := range asked.refused {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
|
||||||
|
len(names))
|
||||||
|
for _, name := range names {
|
||||||
|
fmt.Printf(" %s\n", name)
|
||||||
|
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
||||||
|
fmt.Printf(" %s\n", strings.TrimSpace(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
|
||||||
|
if asked.network != "" {
|
||||||
|
fmt.Printf("the private network could not be computed:\n %s\n\n",
|
||||||
|
strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(wrong) > 0 {
|
||||||
|
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
|
||||||
|
for _, d := range wrong {
|
||||||
|
fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
|
||||||
|
if d.Refused != "" {
|
||||||
|
// The host's own words. It says exactly what it could not accept, and nothing
|
||||||
|
// written here would say it better.
|
||||||
|
fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused))
|
||||||
|
}
|
||||||
|
for _, f := range d.Failed {
|
||||||
|
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
|
||||||
|
}
|
||||||
|
if d.Stuck() {
|
||||||
|
// Said apart from the failure itself. The host's words say what is wrong; this
|
||||||
|
// says it is not new — the machine has applied, failed the same way and reported
|
||||||
|
// so this many times, and will keep doing exactly that until something changes
|
||||||
|
// (novox/hq 04-ISSUES/065).
|
||||||
|
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
|
||||||
|
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(quiet) > 0 {
|
||||||
|
var said []string
|
||||||
|
for _, n := range quiet {
|
||||||
|
said = append(said, n.Name+" ("+heardFrom(n)+")")
|
||||||
|
}
|
||||||
|
fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n",
|
||||||
|
len(quiet), strings.Join(said, "\n "))
|
||||||
|
}
|
||||||
|
|
||||||
|
if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
|
||||||
|
fmt.Printf("%d plan(s) open", len(open))
|
||||||
|
if late > 0 {
|
||||||
|
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
||||||
|
}
|
||||||
|
fmt.Println(":")
|
||||||
|
for _, p := range open {
|
||||||
|
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused))
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(behind) > 0 {
|
||||||
|
var names []string
|
||||||
|
for m := range behind {
|
||||||
|
names = append(names, m)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
|
||||||
|
for _, m := range names {
|
||||||
|
from := sources[m]
|
||||||
|
fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
|
||||||
|
if on := behind[m]; len(on) > 0 {
|
||||||
|
// The part somebody actually wants. A module being out of date is a fact about
|
||||||
|
// the catalogue; machines running the old one is the thing with consequences.
|
||||||
|
fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", "))
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" %-18s assigned to nothing\n", "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The remedy, beside the problem. A status that says what is wrong and not what to do
|
||||||
|
// about it makes somebody go and find the command, and the command is the whole point of
|
||||||
|
// having noticed.
|
||||||
|
fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n")
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(asked.waiting) > 0 {
|
||||||
|
// The other half of "is anything out of date": a module behind its source says the
|
||||||
|
// catalogue is old, and this says a machine is — and only this one has somebody's change
|
||||||
|
// waiting inside it.
|
||||||
|
var told, never []string
|
||||||
|
for _, m := range asked.waiting {
|
||||||
|
if m.Never {
|
||||||
|
never = append(never, m.Node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
told = append(told, m.Node)
|
||||||
|
}
|
||||||
|
if len(told) > 0 {
|
||||||
|
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
|
||||||
|
len(told), strings.Join(told, ", "))
|
||||||
|
}
|
||||||
|
if len(never) > 0 {
|
||||||
|
// Never told is not out of date. The remedy is the same push and the situation is
|
||||||
|
// not the same at all: nobody has ever asked this machine to be anything.
|
||||||
|
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
|
||||||
|
len(never), strings.Join(never, ", "))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `push --behind` sends them\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if split := hostSplit(nodes); len(split) > 1 {
|
||||||
|
// **Before a declaration gains a field, every machine has to understand it** (novox/hq
|
||||||
|
// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
|
||||||
|
// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
|
||||||
|
// no record of which host any machine ran, so that order was kept by somebody remembering it.
|
||||||
|
//
|
||||||
|
// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
|
||||||
|
// commits have no order — the first version of this said "N machines run an older host" and
|
||||||
|
// named the three that were newer, because it compared two hashes as strings. What the mesh
|
||||||
|
// can say truthfully is that the machines do not all run the same host, and which machines
|
||||||
|
// hold which. Ordering needs a version that is ordered, and that is the host's to report.
|
||||||
|
versions := make([]string, 0, len(split))
|
||||||
|
for v := range split {
|
||||||
|
versions = append(versions, v)
|
||||||
|
}
|
||||||
|
sort.Strings(versions)
|
||||||
|
fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
|
||||||
|
for _, v := range versions {
|
||||||
|
sort.Strings(split[v])
|
||||||
|
fmt.Printf(" %-12s %s\n", v, strings.Join(split[v], ", "))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n a host refuses a declaration carrying a field it does not know, whole — so the\n" +
|
||||||
|
" mesh may send only what every one of these understands. Which of them is newer is\n" +
|
||||||
|
" not readable from a commit; that needs a version the host reports as ordered\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(asked.filtered) > 0 {
|
||||||
|
// A converged machine is filtered by the mesh alone, and the mesh says truthfully which
|
||||||
|
// (novox/hq ADR 0168). One that is not — a predecessor's chain still refusing, a found
|
||||||
|
// firewall in force again — is said here, and is not well.
|
||||||
|
machines := make([]string, 0, len(asked.filtered))
|
||||||
|
for name := range asked.filtered {
|
||||||
|
machines = append(machines, name)
|
||||||
|
}
|
||||||
|
sort.Strings(machines)
|
||||||
|
fmt.Printf("%d converged machine(s) are not filtered by the mesh alone:\n", len(machines))
|
||||||
|
for _, name := range machines {
|
||||||
|
f := asked.filtered[name]
|
||||||
|
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||||
|
fmt.Printf(" %-12s the found firewall (%s) is in force again; the next apply retires it\n", name, fw.Kind)
|
||||||
|
}
|
||||||
|
for _, x := range f.Others() {
|
||||||
|
fmt.Printf(" %-12s %s (%s): %s\n", name, x.Where, x.Owner, x.Refuses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("\n the mesh wrote none of these and removes none; `node show <node>` lists every filter with its owner\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(asked.untaken) > 0 {
|
||||||
|
// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
|
||||||
|
// somebody chose and can leave alone; a module assigned to one and never taken is work
|
||||||
|
// outstanding that reads exactly like work finished. That reading is what stopped a
|
||||||
|
// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
|
||||||
|
machines := make([]string, 0, len(asked.untaken))
|
||||||
|
for name := range asked.untaken {
|
||||||
|
machines = append(machines, name)
|
||||||
|
}
|
||||||
|
sort.Strings(machines)
|
||||||
|
total := 0
|
||||||
|
for _, held := range asked.untaken {
|
||||||
|
for _, n := range held {
|
||||||
|
total += n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
|
||||||
|
"taken — so it is running none of what it declares:\n", total)
|
||||||
|
for _, name := range machines {
|
||||||
|
modules := make([]string, 0, len(asked.untaken[name]))
|
||||||
|
for m := range asked.untaken[name] {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
parts := make([]string, 0, len(modules))
|
||||||
|
for _, m := range modules {
|
||||||
|
parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-12s %s\n", name, strings.Join(parts, ", "))
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(asked.unheld) > 0 {
|
||||||
|
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||||
|
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||||
|
// holds, until every machine has its holders and the switch makes it a refusal.
|
||||||
|
fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
|
||||||
|
len(asked.unheld))
|
||||||
|
for _, u := range asked.unheld {
|
||||||
|
holders := "no module in the catalogue claims it yet"
|
||||||
|
if len(u.Holders) > 0 {
|
||||||
|
holders = "could be held by " + strings.Join(u.Holders, ", ")
|
||||||
|
}
|
||||||
|
fmt.Printf(" %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
|
||||||
|
}
|
||||||
|
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||||
|
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||||
|
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||||
|
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
||||||
|
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if asked.well() {
|
||||||
|
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||||
|
// and getting here means every question was asked and answered.
|
||||||
|
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
||||||
|
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||||
|
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||||
|
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||||
|
// about the relationship between the mesh and a machine — applied what it was sent, matches
|
||||||
|
// what would be sent, built from what the source has. None of them asks whether a module can
|
||||||
|
// reach what it requires, and the mesh composes every one of those grants itself.
|
||||||
|
//
|
||||||
|
// Said here rather than left to be inferred. A reader who acts on the line above is acting on
|
||||||
|
// "the machines are as the mesh described them", and the distance between that and "it works"
|
||||||
|
// is where the eleven hours went.
|
||||||
|
fmt.Printf("\n That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
|
||||||
|
" no grant the mesh composed has been tested, so a module unable to reach what it\n" +
|
||||||
|
" requires would not appear above (04-ISSUES/145)\n")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstLine is as much of a failure as belongs in a list.
|
||||||
|
func firstLine(s string) string {
|
||||||
|
if cut := strings.IndexByte(s, '\n'); cut >= 0 {
|
||||||
|
return strings.TrimSpace(s[:cut])
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(s)
|
||||||
|
}
|
||||||
|
|
||||||
|
// builds keeps what a builder said, for the serving control plane.
|
||||||
|
//
|
||||||
|
// A type of its own rather than a method on the enrolment, because they are unrelated things
|
||||||
|
// arriving on one queue and an implementation of one should not have to say anything about the
|
||||||
|
// other.
|
||||||
|
type builds struct {
|
||||||
|
inv *inventory.Inventory
|
||||||
|
open *stores
|
||||||
|
}
|
||||||
|
|
||||||
|
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
|
||||||
|
//
|
||||||
|
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
|
||||||
|
// are three now — a person's status, its JSON, and a page — and three implementations of "which
|
||||||
|
// machine is not doing what it was told" would be three chances to disagree about it.
|
||||||
|
//
|
||||||
|
// The order is the design and not a convenience: is anything broken, is anything not answering, is
|
||||||
|
// anything out of date. The first has consequences now, the second may, the third is a plan for
|
||||||
|
// later — and anything that led with the third would bury the first.
|
||||||
|
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
var out answers
|
||||||
|
var err error
|
||||||
|
|
||||||
|
out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
out.nodes, err = inv.Nodes(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
for _, n := range out.nodes {
|
||||||
|
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
||||||
|
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
||||||
|
// and could not.
|
||||||
|
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
||||||
|
out.quiet = append(out.quiet, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.behind, err = inv.Behind(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
// And why any machine cannot be worked out at all, which is neither of the first two questions
|
||||||
|
// and is asked before them both in practice: a machine nothing can be computed for is not
|
||||||
|
// broken, not quiet and not behind, and every other answer here would call it well.
|
||||||
|
//
|
||||||
|
// Read through whoResolves, which is what the private network is built from, so this and the
|
||||||
|
// network agree about who could not be resolved rather than deciding it twice.
|
||||||
|
_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
// And what each machine is holding rather than running, by the module that would run it. Read
|
||||||
|
// from what the machine itself last reported, not from what take-time computed: the machine is
|
||||||
|
// the only thing that knows what it found (novox/hq 04-ISSUES/125).
|
||||||
|
out.untaken, err = untakenModules(ctx, inv, out.nodes)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
// And which converged machines something other than the mesh filters (novox/hq ADR 0168), as
|
||||||
|
// each last reported — the account that was missing when a predecessor's chain refused what the
|
||||||
|
// mesh declared open for eleven hours (04-ISSUES/144, 145).
|
||||||
|
out.filtered, err = filteredMachines(ctx, inv, out.nodes)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
|
||||||
|
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||||
|
// the private network is built from and should say nothing else; a machine that does not
|
||||||
|
// resolve is already in refused, and is passed over here.
|
||||||
|
for _, n := range out.nodes {
|
||||||
|
plan, _, err := planFor(ctx, open, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
if unresolvable(err) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
out.unheld = append(out.unheld, plan.Unheld...)
|
||||||
|
}
|
||||||
|
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||||
|
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||||
|
|
||||||
|
// And which machines are not running what the mesh would send them. The same question as a
|
||||||
|
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||||
|
// this one says a machine is — and only the second has anybody's change waiting in it.
|
||||||
|
//
|
||||||
|
// **One machine that cannot be resolved must not take the answer away from every other**
|
||||||
|
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
|
||||||
|
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
|
||||||
|
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||||
|
// reason is kept and reported as data; every question that does not depend on it is still
|
||||||
|
// answered.
|
||||||
|
would, err := wouldSend(ctx, open, out.nodes)
|
||||||
|
if err != nil {
|
||||||
|
out.network = err.Error()
|
||||||
|
would = map[string]string{}
|
||||||
|
}
|
||||||
|
out.waiting, err = inv.Waiting(ctx, would)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
out.reported, err = inv.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
out.sources = map[string]inventory.Source{}
|
||||||
|
for module := range out.behind {
|
||||||
|
from, err := inv.SourceOf(ctx, module)
|
||||||
|
if err != nil {
|
||||||
|
return answers{}, err
|
||||||
|
}
|
||||||
|
out.sources[module] = from
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// untakenModules is, per machine, each module whose resources that machine is holding as found, and
|
||||||
|
// how many.
|
||||||
|
//
|
||||||
|
// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
|
||||||
|
// and reports it; the mesh's take-time listing is a different thing and was the one this command used
|
||||||
|
// to have, which is why a module assigned after the listing showed nothing at all
|
||||||
|
// (novox/hq 04-ISSUES/125).
|
||||||
|
//
|
||||||
|
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
|
||||||
|
// the caller prints nothing.
|
||||||
|
// filteredMachines is every converged machine not filtered by the mesh alone, with what it last said
|
||||||
|
// filters it (novox/hq ADR 0168). An adopted machine keeps its found firewall by design and is not
|
||||||
|
// counted; a machine that has not said is not said to be filtered by anything.
|
||||||
|
func filteredMachines(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||||
|
map[string]inventory.Filtering, error) {
|
||||||
|
out := map[string]inventory.Filtering{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.Adopted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
f, err := inv.FilteringOf(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("what filters %s cannot be read: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !f.Alone() {
|
||||||
|
out[n.Name] = f
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||||
|
map[string]map[string]int, error) {
|
||||||
|
|
||||||
|
out := map[string]map[string]int{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
said, err := inv.AdoptionOf(ctx, n.Name)
|
||||||
|
if err != nil {
|
||||||
|
// A machine whose record cannot be read is not a machine holding nothing. Said, because
|
||||||
|
// answering "nothing held" from a failed read is the shape this whole issue is about.
|
||||||
|
return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
|
||||||
|
}
|
||||||
|
for _, h := range said.Held {
|
||||||
|
if h.Module == "" {
|
||||||
|
continue // a hold the mesh cannot attribute to a module has nothing to take
|
||||||
|
}
|
||||||
|
if out[n.Name] == nil {
|
||||||
|
out[n.Name] = map[string]int{}
|
||||||
|
}
|
||||||
|
out[n.Name][h.Module]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// well is whether every question this command asks came back with nothing to say.
|
||||||
|
//
|
||||||
|
// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
|
||||||
|
// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
|
||||||
|
// and is not doing what it was told either.
|
||||||
|
//
|
||||||
|
// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
|
||||||
|
// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
|
||||||
|
// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
|
||||||
|
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||||
|
func (a answers) well() bool {
|
||||||
|
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||||
|
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||||
|
len(a.filtered) == 0 && len(a.unheld) == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// hostSplit is which machines report which host version, for every version more than one machine
|
||||||
|
// could disagree about.
|
||||||
|
//
|
||||||
|
// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
|
||||||
|
// commits have no order. The first version of this returned "the machines behind the newest" by
|
||||||
|
// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
|
||||||
|
// host as the ones behind — an arbitrary lexicographic result presented as a fact
|
||||||
|
// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
|
||||||
|
// that says less, which is the whole subject of 04-ISSUES/145.
|
||||||
|
//
|
||||||
|
// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
|
||||||
|
// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
|
||||||
|
// function's to infer.
|
||||||
|
//
|
||||||
|
// Machines that have not reported a version are left out entirely: they are not a version, and
|
||||||
|
// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
|
||||||
|
func hostSplit(nodes []inventory.Node) map[string][]string {
|
||||||
|
out := map[string][]string{}
|
||||||
|
for _, n := range nodes {
|
||||||
|
if n.HostVersion == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[n.HostVersion] = append(out[n.HostVersion], n.Name)
|
||||||
|
}
|
||||||
|
if len(out) < 2 {
|
||||||
|
return nil // one version, or none reported: nothing to disagree about
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// **One machine's failure must never take the answer away from a reader asking about the others.**
|
||||||
|
//
|
||||||
|
// A blocked machine is not on the private network, and a mesh whose hub is that machine has no hub
|
||||||
|
// — which came back through the reading as a refusal, so `status` printed nothing at all and
|
||||||
|
// `status --json` emitted multi-line prose on stderr and not one byte of JSON. A machine-readable
|
||||||
|
// interface that stops being machine-readable exactly when something is wrong is one nobody can
|
||||||
|
// build an alarm on.
|
||||||
|
func TestOneBlockedMachineDoesNotDestroyTheWholeDocument(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
|
||||||
|
// Break the hub, using nothing but the command a person has.
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{"rival-one", "rival-two"} {
|
||||||
|
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("one blocked machine made the whole mesh unreadable: %v", err)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var parsed map[string]any
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
t.Fatalf("what a script would read is not JSON: %v\n%s", err, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The failure is in the document, as data, on the machine it belongs to.
|
||||||
|
unresolved, _ := parsed["unresolved"].([]any)
|
||||||
|
if len(unresolved) == 0 {
|
||||||
|
t.Fatalf("a machine that cannot be worked out is absent from the document:\n%s", body)
|
||||||
|
}
|
||||||
|
var found bool
|
||||||
|
for _, row := range unresolved {
|
||||||
|
entry, _ := row.(map[string]any)
|
||||||
|
if entry["node"] != "anchor" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
found = true
|
||||||
|
problem, _ := entry["problem"].(string)
|
||||||
|
if !strings.Contains(problem, "the-seat") {
|
||||||
|
t.Errorf("the machine's problem is not its own words: %q", problem)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("the blocked machine is not the one named:\n%s", body)
|
||||||
|
}
|
||||||
|
// And the mesh is still counted, so a reader can tell "none blocked" from "none at all".
|
||||||
|
if parsed["machines"] != float64(2) {
|
||||||
|
t.Errorf("the rest of the document did not survive: %v", parsed["machines"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A well mesh carries the field as an empty list, not as nothing: a reader distinguishing "none
|
||||||
|
// blocked" from "this field is missing" would have to handle both, and null is the one that gets
|
||||||
|
// forgotten.
|
||||||
|
func TestAWellMeshCarriesAnEmptyUnresolvedList(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var parsed map[string]any
|
||||||
|
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
list, ok := parsed["unresolved"].([]any)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("\"unresolved\" is %T, not a list:\n%s", parsed["unresolved"], body)
|
||||||
|
}
|
||||||
|
if len(list) != 0 {
|
||||||
|
t.Fatalf("a well mesh reports blocked machines: %v", list)
|
||||||
|
}
|
||||||
|
if _, said := parsed["network"]; said {
|
||||||
|
t.Errorf("a well mesh says the network could not be computed: %v", parsed["network"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And the page says it too, from the same reading. A board that renders "all well" over a mesh
|
||||||
|
// where nothing can be sent anywhere is worse than a board that is down.
|
||||||
|
func TestThePageSaysWhichMachinesCannotBeWorkedOut(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
one, two := rivals()
|
||||||
|
register(t, open, one)
|
||||||
|
register(t, open, two)
|
||||||
|
for _, m := range []string{"rival-one", "rival-two"} {
|
||||||
|
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
asked, err := theThreeQuestions(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rendered := render(t, viewOf(asked))
|
||||||
|
for _, want := range []string{"Can everything be worked out?", "anchor", "the-seat"} {
|
||||||
|
if !strings.Contains(rendered, want) {
|
||||||
|
t.Fatalf("the page does not say %q:\n%s", want, rendered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,199 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/identity"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/licences"
|
||||||
|
"github.com/novox/mesh-controller/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// reaching each context's store, which no other context may touch.
|
||||||
|
//
|
||||||
|
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||||
|
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||||
|
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||||
|
|
||||||
|
// migrate brings every held context's schema up to date.
|
||||||
|
//
|
||||||
|
// Reported per context and per migration, because this runs during a bootstrap on a machine with
|
||||||
|
// nothing else on it — the output is the only account of what happened, and "migrated" is not one.
|
||||||
|
func migrate(ctx context.Context) error {
|
||||||
|
for _, c := range held {
|
||||||
|
migrations, err := c.migrations()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s, err := store.Open(ctx, c.name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
|
||||||
|
// The bootstrap raises PostgreSQL moments before this runs, and a container that is
|
||||||
|
// running is not a database that will answer — a distinction this project has already
|
||||||
|
// paid for once, when a crash-looping database reported itself as up between restarts.
|
||||||
|
if err := s.Ready(ctx, 60*time.Second); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
done, err := s.Migrate(ctx, migrations)
|
||||||
|
for _, m := range done {
|
||||||
|
fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(done) == 0 {
|
||||||
|
applied, err := s.AppliedMigrations(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The modules the control plane ships with itself. Recorded here rather than by hand, because
|
||||||
|
// a mesh whose own private network is missing from the catalogue would have nothing to assign
|
||||||
|
// and no way to say why.
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
for _, m := range provided {
|
||||||
|
if err := inv.Provide(ctx, m); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("provided %s\n", m.Module)
|
||||||
|
}
|
||||||
|
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
||||||
|
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
||||||
|
// operator's changes in the table as they are.
|
||||||
|
added, err := inv.SeedSeats(ctx, catalogue.DefaultSeats())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("seeded %d seat(s)\n", added)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// openInventory connects and waits, the way every command that touches it needs to.
|
||||||
|
func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||||
|
inv, err := inventory.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := inv.Ready(ctx, 30*time.Second); err != nil {
|
||||||
|
inv.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||||
|
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||||
|
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||||
|
// defaults in force: the set is never emptied by a read that found nothing, which would refuse
|
||||||
|
// every claim. So this can only ever replace the defaults with what the mesh actually holds.
|
||||||
|
if seats, err := inv.Seats(ctx); err == nil {
|
||||||
|
catalogue.UseSeats(seats)
|
||||||
|
}
|
||||||
|
// And the former names, so a reference to a seat's old name resolves after a rename (ADR 0122).
|
||||||
|
if aliases, err := inv.Aliases(ctx); err == nil {
|
||||||
|
catalogue.UseAliases(aliases)
|
||||||
|
}
|
||||||
|
return inv, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func openIdentity(ctx context.Context) (*identity.Identity, error) {
|
||||||
|
ident, err := identity.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ident.Ready(ctx, 30*time.Second); err != nil {
|
||||||
|
ident.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ident, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// openLicences connects to the context that holds which model access exists and who may use it.
|
||||||
|
func openLicences(ctx context.Context) (*licences.Licences, error) {
|
||||||
|
held, err := licences.Open(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := held.Ready(ctx, 30*time.Second); err != nil {
|
||||||
|
held.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return held, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// stores is what one command has open.
|
||||||
|
//
|
||||||
|
// **Opened once, not once per machine.** Working out what a machine should be reaches the identity
|
||||||
|
// context for its certificate and the licence context for its model access, and both were opened —
|
||||||
|
// and waited on — inside functions called for every node in a push. Two machines hid it; fifty
|
||||||
|
// would be fifty connect-and-wait cycles for data that does not change while the push runs.
|
||||||
|
//
|
||||||
|
// Each is opened on first use rather than up front, because most commands need one context and
|
||||||
|
// paying to reach three would be the same waste from the other side.
|
||||||
|
type stores struct {
|
||||||
|
inventory *inventory.Inventory
|
||||||
|
identity *identity.Identity
|
||||||
|
licences *licences.Licences
|
||||||
|
}
|
||||||
|
|
||||||
|
// open connects to the inventory, which every command that touches the mesh needs.
|
||||||
|
func openStores(ctx context.Context) (*stores, error) {
|
||||||
|
inv, err := openInventory(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &stores{inventory: inv}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Identity is this control plane's own identity context, opened if it has not been.
|
||||||
|
func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
|
||||||
|
if h.identity != nil {
|
||||||
|
return h.identity, nil
|
||||||
|
}
|
||||||
|
opened, err := openIdentity(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
h.identity = opened
|
||||||
|
return opened, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Licences is the context holding model access, opened if it has not been.
|
||||||
|
func (h *stores) Licences(ctx context.Context) (*licences.Licences, error) {
|
||||||
|
if h.licences != nil {
|
||||||
|
return h.licences, nil
|
||||||
|
}
|
||||||
|
opened, err := openLicences(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
h.licences = opened
|
||||||
|
return opened, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close lets go of everything that was opened, in any order: they are separate connections to
|
||||||
|
// separate databases and none of them knows about the others.
|
||||||
|
func (h *stores) Close() {
|
||||||
|
if h.licences != nil {
|
||||||
|
h.licences.Close()
|
||||||
|
}
|
||||||
|
if h.identity != nil {
|
||||||
|
h.identity.Close()
|
||||||
|
}
|
||||||
|
if h.inventory != nil {
|
||||||
|
h.inventory.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
|
||||||
|
// and branch had not built, and closes them as superseded; another repository's plan, another
|
||||||
|
// branch's, and a plan made after it are left alone.
|
||||||
|
func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
|
||||||
|
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||||
|
sent := at.Add(time.Minute)
|
||||||
|
plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||||
|
return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
|
||||||
|
Created: created, State: inventory.PlanRolling, Modules: modules}
|
||||||
|
}
|
||||||
|
older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
|
||||||
|
"gitea": {State: "built", SentAt: &sent}, // done with: stays done
|
||||||
|
"keycloak": {State: "asked"}, // asked, not answered: folded
|
||||||
|
"plex": {}, // not yet asked: folded
|
||||||
|
"agent": {State: "built"}, // built, rolls out, not sent: folded
|
||||||
|
"notes": {State: "built"}, // built, records: nothing to send
|
||||||
|
})
|
||||||
|
stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
|
||||||
|
"runtime": {State: "asked"},
|
||||||
|
})
|
||||||
|
other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
|
||||||
|
release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
|
||||||
|
later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
|
||||||
|
done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
|
||||||
|
done.State = inventory.PlanDone
|
||||||
|
|
||||||
|
newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
|
||||||
|
newer.Commit = "97b1b2b0c0ffee"
|
||||||
|
rollsOut := func(m string) bool { return m != "notes" }
|
||||||
|
folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
|
||||||
|
|
||||||
|
if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
|
||||||
|
t.Fatalf("folded %v, wanted %v", folded, want)
|
||||||
|
}
|
||||||
|
var ids []string
|
||||||
|
for _, p := range closed {
|
||||||
|
ids = append(ids, p.ID)
|
||||||
|
if p.State != inventory.PlanSuperseded || p.Open() {
|
||||||
|
t.Errorf("%s was left %s", p.ID, p.State)
|
||||||
|
}
|
||||||
|
if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
|
||||||
|
t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
|
||||||
|
t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
|
||||||
|
"finished one are left alone", ids, want)
|
||||||
|
}
|
||||||
|
if other.State != inventory.PlanRolling {
|
||||||
|
t.Fatal("the plan handed in was changed in place")
|
||||||
|
}
|
||||||
|
if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
|
||||||
|
t.Fatalf("a superseded plan reads %q", line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// novox/hq issue 254: a person closes a plan that will not move again, by its id.
|
||||||
|
func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||||
|
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||||
|
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||||
|
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||||
|
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||||
|
}
|
||||||
|
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||||
|
t.Fatal("a plan already closed was closed again")
|
||||||
|
}
|
||||||
|
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||||
|
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||||
|
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the forge's take compares, as a machine would report it.
|
||||||
|
func aForgeComparison() comparison {
|
||||||
|
return comparison{reported: inventory.Adoption{
|
||||||
|
Firewall: "ufw",
|
||||||
|
Held: []inventory.Held{
|
||||||
|
{ID: "forge.server", Module: "forge", Kind: "container", Target: "forge", Facts: map[string]any{
|
||||||
|
"image": "forge:1.27.3", "image_created": "2026-09-17T10:00:00Z",
|
||||||
|
"declared_image": "forge:1.22.6", "declared_image_created": "2026-08-20T10:00:00Z", "downgrade": true,
|
||||||
|
"networks": map[string]any{"predecessor_default": []any{"office", "db"}},
|
||||||
|
"ports": []any{"3000/tcp>0.0.0.0:3000"}, "declared_ports": []any{"3000:3000"},
|
||||||
|
}},
|
||||||
|
{ID: "forge.config", Module: "forge", Kind: "file", Target: "/etc/forge/app.ini", Kept: "/var/lib/mesh/kept/app.ini",
|
||||||
|
Facts: map[string]any{"differs": true, "difference": []any{"- private scope: local", "+ upstream: public"}}},
|
||||||
|
{ID: "other.server", Module: "other", Kind: "container", Target: "other"},
|
||||||
|
},
|
||||||
|
Reachable: []inventory.Reach{
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 3000, By: "forge", Published: true, ContainerPort: 3000},
|
||||||
|
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A take is a comparison (novox/hq ADR 0163): the preview puts what runs beside what the module
|
||||||
|
// declares, and an older image or a differing file refuses unless named.
|
||||||
|
func TestATakePreviewsTheComparisonAndRefusesWhatIsNotNamed(t *testing.T) {
|
||||||
|
c := aForgeComparison()
|
||||||
|
preview, refusals, saw := comparisonOf("forge", c, takeOptions{})
|
||||||
|
for _, want := range []string{"runs forge:1.27.3 (made 2026-09-17)", "declares forge:1.22.6 (made 2026-08-20)", "DOWNGRADE",
|
||||||
|
"on the network predecessor_default with office, db", "will not once it moves to the module's own network",
|
||||||
|
"publishes 3000/tcp>0.0.0.0:3000; the module declares 3000:3000",
|
||||||
|
// How far the port reaches now, as the machine reported it (rule 1).
|
||||||
|
"reachable now at 0.0.0.0:3000 (tcp, container port 3000), behind the found firewall (ufw)",
|
||||||
|
"- private scope: local", "original kept at /var/lib/mesh/kept/app.ini"} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(preview, "other") {
|
||||||
|
t.Errorf("another module's held things are in the preview:\n%s", preview)
|
||||||
|
}
|
||||||
|
if len(refusals) != 2 || !strings.Contains(refusals[0], "--downgrade") || !strings.Contains(refusals[1], "--replace /etc/forge/app.ini") {
|
||||||
|
t.Fatalf("the downgrade and the differing file refuse, each naming its override: %v", refusals)
|
||||||
|
}
|
||||||
|
if len(saw) != 12 {
|
||||||
|
t.Fatalf("the preview's digest is %q", saw)
|
||||||
|
}
|
||||||
|
// Named, they pass.
|
||||||
|
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"/etc/forge/app.ini": true}}); len(refusals) != 0 {
|
||||||
|
t.Fatalf("named differences still refused: %v", refusals)
|
||||||
|
}
|
||||||
|
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
|
||||||
|
t.Fatalf("replace * did not cover the file: %v", refusals)
|
||||||
|
}
|
||||||
|
// A held thing with no facts yet — a host older than this — refuses nothing and says what it can.
|
||||||
|
if preview, refusals, _ := comparisonOf("other", c, takeOptions{}); len(refusals) != 0 || !strings.Contains(preview, "container other") {
|
||||||
|
t.Fatalf("a factless hold: %q %v", preview, refusals)
|
||||||
|
}
|
||||||
|
// The digest is of what the preview says: a fact changing changes it.
|
||||||
|
c.reported.Held[0].Facts["image"] = "forge:1.27.4"
|
||||||
|
if _, _, again := comparisonOf("forge", c, takeOptions{}); again == saw {
|
||||||
|
t.Fatal("the found image changed and the digest did not")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A secret the mesh minted for a service whose data was found refuses: the running service already
|
||||||
|
// has a value (rule 2). Accepted, it is carried in; `--mint` says the service shall take the new one.
|
||||||
|
func TestAMintedSecretForFoundDataRefusesUnlessAcceptedOrMinted(t *testing.T) {
|
||||||
|
c := aForgeComparison()
|
||||||
|
c.secrets = []inventory.SecretState{
|
||||||
|
{Name: "admin", Origin: inventory.OriginMade},
|
||||||
|
{Name: "postgres-database", Origin: inventory.OriginMade, Provider: "anchor"},
|
||||||
|
{Name: "broker", Origin: inventory.OriginAccepted},
|
||||||
|
}
|
||||||
|
preview, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||||
|
for _, want := range []string{
|
||||||
|
"own secret admin: MINTED by the mesh and not accepted",
|
||||||
|
"secret from anchor postgres-database: MINTED by the mesh and not accepted",
|
||||||
|
"own secret broker: accepted from a person, carried in as it is",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(refusals) != 2 {
|
||||||
|
t.Fatalf("two minted secrets refuse: %v", refusals)
|
||||||
|
}
|
||||||
|
if !strings.Contains(refusals[0], "`secret accept <node> forge admin`") || !strings.Contains(refusals[0], "`--mint admin`") {
|
||||||
|
t.Errorf("the own secret's refusal names accepting it and minting it: %s", refusals[0])
|
||||||
|
}
|
||||||
|
if !strings.Contains(refusals[1], "`secret accept <node> forge postgres-database --provider anchor`") {
|
||||||
|
t.Errorf("the required secret's refusal names its provider: %s", refusals[1])
|
||||||
|
}
|
||||||
|
preview, refusals, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true},
|
||||||
|
Mint: map[string]bool{"admin": true, "postgres-database": true}})
|
||||||
|
if len(refusals) != 0 || !strings.Contains(preview, "admin: minted by the mesh; the service takes the new value, as --mint said") {
|
||||||
|
t.Fatalf("--mint did not pass the minted secrets: %v\n%s", refusals, preview)
|
||||||
|
}
|
||||||
|
// With no found data — only a file held — the service has no value of its own, and a minted
|
||||||
|
// secret is simply said.
|
||||||
|
c.reported.Held = c.reported.Held[1:2]
|
||||||
|
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
|
||||||
|
t.Fatalf("a minted secret refused with no data found: %v", refusals)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A found network a per-machine setting keeps is named in the preview (rule 4), and the module's
|
||||||
|
// settings are said with where each came from, composed or not (rules 1 and 6).
|
||||||
|
func TestTheKeptNetworkAndTheSettingsAreInThePreview(t *testing.T) {
|
||||||
|
c := aForgeComparison()
|
||||||
|
c.keeps = map[string][]string{"forge.server": {"predecessor_default"}}
|
||||||
|
c.layers = []catalogue.Layer{
|
||||||
|
{From: catalogue.MeshWideLayer, Values: map[string]any{"site": "x"}},
|
||||||
|
{From: "anchor", Values: map[string]any{catalogue.NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}},
|
||||||
|
}
|
||||||
|
preview, _, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||||
|
for _, want := range []string{
|
||||||
|
"on the network predecessor_default with office, db — kept by this machine's setting, so they still reach it by name once taken",
|
||||||
|
"settings from the mesh: site",
|
||||||
|
"settings from anchor: networks",
|
||||||
|
} {
|
||||||
|
if !strings.Contains(preview, want) {
|
||||||
|
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.Contains(preview, "will not once it moves") {
|
||||||
|
t.Errorf("a kept network is still said to be lost:\n%s", preview)
|
||||||
|
}
|
||||||
|
c.settingsRefused = "forge: ports is a { port: machine-port } map"
|
||||||
|
preview, _, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||||
|
if !strings.Contains(preview, "SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: forge: ports") {
|
||||||
|
t.Errorf("settings that cannot compose are not said:\n%s", preview)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
|
||||||
|
// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
|
||||||
|
// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
|
||||||
|
|
||||||
|
func aContainer(name string) catalogue.Manifest {
|
||||||
|
return catalogue.Manifest{Module: name, Version: "1",
|
||||||
|
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
register(t, open, aContainer("web"))
|
||||||
|
register(t, open, aContainer("db"))
|
||||||
|
// anchor already lacks a runtime for db: true, and not this act's to say.
|
||||||
|
if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
said, err := assign(ctx, open, "laptop", "web")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%v\n%s", err, said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) {
|
||||||
|
t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said)
|
||||||
|
}
|
||||||
|
for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} {
|
||||||
|
if strings.Contains(said, not) {
|
||||||
|
t.Errorf("the assignment says %q, which it did not change:\n%s", not, said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
|
||||||
|
shelf := map[string]catalogue.Manifest{
|
||||||
|
"web": aContainer("web"),
|
||||||
|
"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
|
||||||
|
Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
|
||||||
|
}
|
||||||
|
lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
|
||||||
|
if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
|
||||||
|
t.Errorf("meeting a dependency said %v", lines)
|
||||||
|
}
|
||||||
|
// Taking the dependent off says nothing: the dependency went with its module.
|
||||||
|
if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
|
||||||
|
t.Errorf("unassigning the dependent said %v", lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
|
||||||
|
unheld := map[string][]catalogue.Unheld{
|
||||||
|
"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
|
||||||
|
"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
|
||||||
|
{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
|
||||||
|
}
|
||||||
|
var named bytes.Buffer
|
||||||
|
reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
|
||||||
|
got := named.String()
|
||||||
|
if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
|
||||||
|
!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
|
||||||
|
t.Errorf("a named push said:\n%s", got)
|
||||||
|
}
|
||||||
|
var all bytes.Buffer
|
||||||
|
reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
|
||||||
|
want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
|
||||||
|
if all.String() != want {
|
||||||
|
t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
|
||||||
|
read := func(f func()) string {
|
||||||
|
old := os.Stderr
|
||||||
|
r, w, _ := os.Pipe()
|
||||||
|
os.Stderr = w
|
||||||
|
f()
|
||||||
|
_ = w.Close()
|
||||||
|
os.Stderr = old
|
||||||
|
var b bytes.Buffer
|
||||||
|
_, _ = b.ReadFrom(r)
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||||
|
t.Errorf("a command logged:\n%s", got)
|
||||||
|
}
|
||||||
|
logUnheldChanges = true
|
||||||
|
defer func() { logUnheldChanges = false }()
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
|
||||||
|
t.Errorf("the serving controller did not log a change:\n%s", got)
|
||||||
|
}
|
||||||
|
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||||
|
t.Errorf("an unchanged report was logged again:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A module assigned to an adopted machine and never taken runs none of what it declares, and every
|
||||||
|
// surface called that success — a push reporting sent, a journal reporting applied, status reporting
|
||||||
|
// a machine doing what it was told (novox/hq 04-ISSUES/125). The holds were only ever in the
|
||||||
|
// machine's own state file.
|
||||||
|
|
||||||
|
// heldOn makes a machine report that it is holding resources for a module, the way an adopted node
|
||||||
|
// does after an apply.
|
||||||
|
func heldOn(t *testing.T, open *stores, node, module string, ids ...string) {
|
||||||
|
t.Helper()
|
||||||
|
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
held := make([]inventory.Held, 0, len(ids))
|
||||||
|
for _, id := range ids {
|
||||||
|
held = append(held, inventory.Held{ID: id, Module: module, Kind: "container", Target: id})
|
||||||
|
}
|
||||||
|
if err := open.inventory.RecordAdoption(t.Context(), record.ID, held, "ufw", nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server")
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := asked.untaken["anchor"]["route-proxy"]; got != 3 {
|
||||||
|
t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) {
|
||||||
|
// The whole of the fault. "all doing what they were told" was true throughout the outage, and
|
||||||
|
// true is not the same as safe to act on: the machine was doing what it was told, and what it
|
||||||
|
// was told had not started. Asserted against the production condition, not a copy of it.
|
||||||
|
quiet := answers{}
|
||||||
|
if !quiet.well() {
|
||||||
|
t.Fatal("a mesh with nothing to say does not read as well, so nothing below means anything")
|
||||||
|
}
|
||||||
|
holding := answers{untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}}}
|
||||||
|
if holding.well() {
|
||||||
|
t.Fatal("a machine holding a module's resources still reads as doing what it was told, " +
|
||||||
|
"which is the sentence that cost every public name on the machine")
|
||||||
|
}
|
||||||
|
// And being adopted does not suppress it: that is a mode somebody chose, not work outstanding.
|
||||||
|
// Kept as an assertion so the difference between the two is deliberate rather than incidental.
|
||||||
|
if !quiet.well() {
|
||||||
|
t.Fatal("the well condition is not stable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) {
|
||||||
|
// End to end through the store, so the condition above is reached by real data and not only by
|
||||||
|
// a constructed value: the machine reports, the mesh records, status asks.
|
||||||
|
open := aMesh(t)
|
||||||
|
heldOn(t, open, "anchor", "route-proxy", "ca", "server")
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(asked.untaken) == 0 {
|
||||||
|
t.Fatal("what the machine reported holding did not reach status")
|
||||||
|
}
|
||||||
|
if asked.well() {
|
||||||
|
t.Fatal("a mesh whose machine reported holds reads as well")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) {
|
||||||
|
open := aMesh(t)
|
||||||
|
heldOn(t, open, "anchor", "route-proxy", "ca", "certs")
|
||||||
|
|
||||||
|
asked, err := theThreeQuestions(t.Context(), open)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := statusAsJSON(asked)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var doc struct {
|
||||||
|
Untaken []struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Held int `json:"held"`
|
||||||
|
} `json:"untaken"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &doc); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(doc.Untaken) != 1 {
|
||||||
|
t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body)
|
||||||
|
}
|
||||||
|
row := doc.Untaken[0]
|
||||||
|
if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 {
|
||||||
|
t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row)
|
||||||
|
}
|
||||||
|
// Absent rather than empty when nothing is held, so a well mesh's document does not carry a
|
||||||
|
// field a reader has to interpret.
|
||||||
|
clean, err := statusAsJSON(answers{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(clean), "untaken") {
|
||||||
|
t.Fatalf("a mesh holding nothing still names untaken: %s", clean)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,721 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
"github.com/novox/mesh-controller/internal/link"
|
||||||
|
)
|
||||||
|
|
||||||
|
// following acts on what the catalogue announces.
|
||||||
|
//
|
||||||
|
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
|
||||||
|
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
|
||||||
|
// the control plane.
|
||||||
|
type following struct{ open *stores }
|
||||||
|
|
||||||
|
// Upgraded sends the machines running a module the version the catalogue now considers current —
|
||||||
|
// or records that they are behind, which is the default and needs no record.
|
||||||
|
//
|
||||||
|
// **Recording is not a second code path.** A machine that is not running what the mesh would send
|
||||||
|
// it is already something the mesh notices and reports; that is what `status` and `push --behind`
|
||||||
|
// are built on. So "record it" is the absence of an action, and the only thing this has to decide
|
||||||
|
// is whether to act.
|
||||||
|
func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||||
|
inv := f.open.inventory
|
||||||
|
|
||||||
|
// The store read first, and an outage there said as one, so the announcement is held and asked
|
||||||
|
// again (novox/hq issue 083). Only here: a push that fails further down is not asked again.
|
||||||
|
decision, err := inv.UpgradeOf(ctx, u.Module)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
on, err := inv.Running(ctx, u.Module)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
if len(on) == 0 {
|
||||||
|
fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !decision.RollOut {
|
||||||
|
// Named rather than counted, and said even though nothing happens: an upgrade that was
|
||||||
|
// deliberately not rolled out and an upgrade that was never noticed look identical in a
|
||||||
|
// log that only speaks when it acts.
|
||||||
|
fmt.Printf("%s moved to %s; %s %s behind it, and this mesh records upgrades rather than "+
|
||||||
|
"rolling them out — `push --behind` when you want them\n",
|
||||||
|
u.Module, shortCommit(u.Commit), readableList(on), isAre(len(on)))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR
|
||||||
|
// 0218). A merge's plan builds the module and sends it one machine first, the rest once that one
|
||||||
|
// has applied it; this announcement arrives as the build registers, and sending here too — one
|
||||||
|
// machine after another without waiting for any to apply — put the new bundle on every machine in
|
||||||
|
// the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by
|
||||||
|
// hand) is still this handler's.
|
||||||
|
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
|
||||||
|
// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
|
||||||
|
// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
|
||||||
|
fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
|
||||||
|
"`status` lists them as behind and `push --behind` sends it\n",
|
||||||
|
u.Module, shortCommit(u.Commit), id, readableList(on))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if decision.Together {
|
||||||
|
fmt.Printf("%s moved to %s; sending %s together\n",
|
||||||
|
u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
|
return askAgainOnGrants(sendTo(ctx, f.open, on))
|
||||||
|
}
|
||||||
|
// One at a time, and stopping at the first that fails.
|
||||||
|
//
|
||||||
|
// **Stopping is the point.** The machines are done one after another precisely so that a
|
||||||
|
// version that breaks the first one does not reach the rest; carrying on past a failure would
|
||||||
|
// make this the same as sending them together, only slower.
|
||||||
|
fmt.Printf("%s moved to %s; sending %s one at a time\n",
|
||||||
|
u.Module, shortCommit(u.Commit), readableList(on))
|
||||||
|
for _, node := range on {
|
||||||
|
if err := sendTo(ctx, f.open, []string{node}); err != nil {
|
||||||
|
return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
||||||
|
node, u.Module, err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
|
||||||
|
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
|
||||||
|
// rather than taken as handled with the machines left on the old version.
|
||||||
|
func askAgainOnGrants(err error) error {
|
||||||
|
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
|
||||||
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
|
||||||
|
// the module that has not finished sending it — or empty when none will (novox/hq issue 249).
|
||||||
|
func rolledOutByAPlan(plans []inventory.Plan, module string) string {
|
||||||
|
for _, p := range plans {
|
||||||
|
if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// readableList names machines the way a sentence does, because this is read by a person deciding
|
||||||
|
// whether an upgrade went where they expected.
|
||||||
|
func readableList(names []string) string {
|
||||||
|
switch len(names) {
|
||||||
|
case 0:
|
||||||
|
return "nothing"
|
||||||
|
case 1:
|
||||||
|
return names[0]
|
||||||
|
case 2:
|
||||||
|
return names[0] + " and " + names[1]
|
||||||
|
}
|
||||||
|
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
func shortCommit(commit string) string {
|
||||||
|
if len(commit) > 8 {
|
||||||
|
return commit[:8]
|
||||||
|
}
|
||||||
|
return commit
|
||||||
|
}
|
||||||
|
|
||||||
|
func isAre(n int) string {
|
||||||
|
if n == 1 {
|
||||||
|
return "is"
|
||||||
|
}
|
||||||
|
return "are"
|
||||||
|
}
|
||||||
|
|
||||||
|
// upgradeCommand says what should happen when a module's current version moves.
|
||||||
|
func upgradeCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
|
||||||
|
together := set.Bool("together", false,
|
||||||
|
"send every machine running it at once, instead of one after another")
|
||||||
|
positionals, err := parseAround(set, args)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(positionals) == 0 {
|
||||||
|
return errors.New("upgrade <module> [roll-out|record] [--together]")
|
||||||
|
}
|
||||||
|
module := positionals[0]
|
||||||
|
|
||||||
|
open, err := openStores(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer open.Close()
|
||||||
|
inv := open.inventory
|
||||||
|
|
||||||
|
if len(positionals) == 1 {
|
||||||
|
decision, err := inv.UpgradeOf(ctx, module)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(sayUpgrade(module, decision))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var decision inventory.Upgrade
|
||||||
|
switch positionals[1] {
|
||||||
|
case "roll-out":
|
||||||
|
decision = inventory.Upgrade{RollOut: true, Together: *together}
|
||||||
|
case "record":
|
||||||
|
if *together {
|
||||||
|
// Refused rather than ignored: --together only means anything for a roll-out, and
|
||||||
|
// accepting it here would store a preference that never applies and looks like it does.
|
||||||
|
return errors.New("`--together` says how to roll out, so it cannot be given with " +
|
||||||
|
"`record`, which is the choice not to")
|
||||||
|
}
|
||||||
|
decision = inventory.Upgrade{}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("upgrade <module> roll-out|record — not %q", positionals[1])
|
||||||
|
}
|
||||||
|
if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Println(sayUpgrade(module, decision))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||||
|
if !u.RollOut {
|
||||||
|
return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
|
||||||
|
"reported as behind", module)
|
||||||
|
}
|
||||||
|
if u.Together {
|
||||||
|
return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
|
||||||
|
"together", module)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("when %s moves, the machines running it are sent the new version one at "+
|
||||||
|
"a time, stopping at the first that fails", module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Announceable is every build this mesh recorded, in the shape the builder announces one.
|
||||||
|
//
|
||||||
|
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
|
||||||
|
// (novox/hq 04-ISSUES/050). A durable queue keeps what arrived after it existed, so a running
|
||||||
|
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
|
||||||
|
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
|
||||||
|
// store the catalogue runs on, and the catalogue itself.
|
||||||
|
//
|
||||||
|
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
|
||||||
|
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
|
||||||
|
// the store's address, so a replay composes the address back in — the store's address as the
|
||||||
|
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
|
||||||
|
// store on the network yet, the recorded form goes as it is.
|
||||||
|
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
|
||||||
|
builds, err := f.open.inventory.Announceable(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out := make([]link.Announcement, 0, len(builds))
|
||||||
|
for _, b := range builds {
|
||||||
|
a := link.Announcement{
|
||||||
|
Module: b.Module, Commit: b.Commit, Repository: b.Repository,
|
||||||
|
Path: b.Path, Ref: b.Ref, Against: b.Against,
|
||||||
|
}
|
||||||
|
if len(b.Manifest) > 0 {
|
||||||
|
a.Manifest = b.Manifest
|
||||||
|
if address != "" {
|
||||||
|
a.Manifest = routedManifest(b.Manifest, b.Made, address)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, made := range routedArtifacts(b.Made, address) {
|
||||||
|
a.Made = append(a.Made, link.MadeArtifact{
|
||||||
|
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
out = append(out, a)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// notNow marks a store that could not be read right now, so the announcement is held rather than
|
||||||
|
// lost; anything else is returned as it was.
|
||||||
|
func notNow(err error) error {
|
||||||
|
if inventory.Unreachable(err) {
|
||||||
|
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SourceMoved is the forge announcing a merge: every module recorded as built from that
|
||||||
|
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
|
||||||
|
// module that stands on another's artifact is built after it and not against the old one
|
||||||
|
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||||
|
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||||
|
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||||
|
inv := f.open.inventory
|
||||||
|
entries, err := inv.Catalogued(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
read, err := inv.ReadRepositories(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||||
|
//
|
||||||
|
// A module **built from** this repository and branch has moved: the mesh records the new commit
|
||||||
|
// as what its source now has, and only what the merge actually changed is rebuilt. A module that
|
||||||
|
// only **packages source from** it has not moved — its own source is somewhere else, at the
|
||||||
|
// commit it already records — so it is rebuilt and its record left alone. Writing this commit as
|
||||||
|
// its source would make it permanently behind a repository its manifest does not come from.
|
||||||
|
var from, packaging []inventory.Entry
|
||||||
|
already := 0
|
||||||
|
for _, e := range entries {
|
||||||
|
switch {
|
||||||
|
case sourceIs(e.Source, m):
|
||||||
|
if e.Source.BuiltFrom == m.Commit {
|
||||||
|
already++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||||
|
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||||
|
// the recorded head backwards and rebuild everything built from that repository, once
|
||||||
|
// per old merge (2026-09-28).
|
||||||
|
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
from = append(from, e)
|
||||||
|
case readsFrom(read[e.Manifest.Module], m):
|
||||||
|
packaging = append(packaging, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(from) == 0 && len(packaging) == 0 {
|
||||||
|
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||||
|
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||||
|
if already > 0 {
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
||||||
|
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
|
||||||
|
m.Owner, m.Repo, m.Base, m.Commit)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// The same judgement for the packaging kind, against the newest look at that repository by
|
||||||
|
// anything built from it: they keep no record of it themselves, and a replayed old merge should
|
||||||
|
// not rebuild them either.
|
||||||
|
if isHistory(m.MergedAt, lastLookAt(entries, m)) {
|
||||||
|
packaging = nil
|
||||||
|
}
|
||||||
|
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||||
|
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||||
|
for _, e := range entries {
|
||||||
|
if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) {
|
||||||
|
fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n",
|
||||||
|
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
touched := whatTheMergeTouched(from, entries, m)
|
||||||
|
for _, e := range touched {
|
||||||
|
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
||||||
|
if len(moved) == 0 {
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||||
|
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// A merge produces a plan the mesh keeps (novox/hq ADR 0162): what moved and everything that
|
||||||
|
// depends on it, along the catalogue's one dependency relation, sorted into tiers. The plan is
|
||||||
|
// written before any build is asked; the first tier is asked; this returns. Outcomes advance it.
|
||||||
|
edges, err := inv.Dependencies(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
var movedNames []string
|
||||||
|
for _, e := range moved {
|
||||||
|
movedNames = append(movedNames, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
|
||||||
|
// another controller reading it between the two would ask the tier again.
|
||||||
|
release, err := inv.HoldPlans(ctx, true)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
plan := planOfMerge(m, movedNames, edges)
|
||||||
|
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
||||||
|
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
||||||
|
// works a repository's modules at a time and a stuck one ends at the next merge.
|
||||||
|
working, err := inv.OpenPlans(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
rollsOut := func(module string) bool {
|
||||||
|
u, err := inv.UpgradeOf(ctx, module)
|
||||||
|
return err == nil && u.RollOut
|
||||||
|
}
|
||||||
|
folded, superseded := supersededBy(plan, working, rollsOut)
|
||||||
|
if len(folded) > 0 {
|
||||||
|
held := map[string]bool{}
|
||||||
|
for _, e := range entries {
|
||||||
|
held[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, name := range movedNames {
|
||||||
|
names[name] = true
|
||||||
|
}
|
||||||
|
var also []string
|
||||||
|
for _, name := range folded {
|
||||||
|
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
||||||
|
// merge's to build.
|
||||||
|
if held[name] && !names[name] {
|
||||||
|
names[name] = true
|
||||||
|
movedNames = append(movedNames, name)
|
||||||
|
also = append(also, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(also) > 0 {
|
||||||
|
again := planOfMerge(m, movedNames, edges)
|
||||||
|
again.ID, again.Created = plan.ID, plan.Created
|
||||||
|
plan = again
|
||||||
|
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
||||||
|
strings.Join(also, ", "), plan.Repository)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hasCycle(plan.Tiers, edges) {
|
||||||
|
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||||
|
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||||
|
// both open, which the next merge settles, rather than neither.
|
||||||
|
for _, old := range superseded {
|
||||||
|
if err := inv.SavePlan(ctx, old); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||||
|
}
|
||||||
|
var tiers []string
|
||||||
|
for i, t := range plan.Tiers {
|
||||||
|
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||||
|
}
|
||||||
|
fmt.Printf("%s/%s merged into %s (%.8s); plan %s, %d module(s) in %d tier(s)\n %s\n",
|
||||||
|
m.Owner, m.Repo, m.Base, m.Commit, plan.ID, len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
|
||||||
|
if len(packaging) > 0 {
|
||||||
|
var also []string
|
||||||
|
for _, e := range packaging {
|
||||||
|
also = append(also, e.Manifest.Module)
|
||||||
|
}
|
||||||
|
fmt.Printf(" %s package source from it, so they are rebuilt and their own source record "+
|
||||||
|
"is left where it is\n", strings.Join(also, ", "))
|
||||||
|
}
|
||||||
|
if err := askTier(ctx, inv, &plan); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||||
|
return notNow(err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||||
|
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||||
|
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||||
|
// branch of the forge's default means.
|
||||||
|
func sourceIs(s inventory.Source, m link.SourceMoved) bool {
|
||||||
|
if !sameRepository(s.Repository, m) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ref := followedBranch(s.Ref)
|
||||||
|
return ref == "" || ref == m.Base
|
||||||
|
}
|
||||||
|
|
||||||
|
// commitRef is a ref that names a commit rather than a branch: what `build --ref <commit>` asks for.
|
||||||
|
var commitRef = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||||||
|
|
||||||
|
// followedBranch is the branch a recorded ref means a module follows (novox/hq 04-ISSUES/215). **A
|
||||||
|
// commit is never a branch to follow.** A build asked at a commit — to try one, or to pin it during a
|
||||||
|
// fix — recorded that commit as the module's ref; every merge after it then failed to match the
|
||||||
|
// module, its plan left it out without saying so, and every plan that rebuilt it asked for that same
|
||||||
|
// old commit again. A commit recorded so is read as the repository's default branch, which is what
|
||||||
|
// the module followed before it; a branch is followed as named.
|
||||||
|
func followedBranch(ref string) string {
|
||||||
|
if commitRef.MatchString(strings.TrimSpace(ref)) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return ref
|
||||||
|
}
|
||||||
|
|
||||||
|
// sameRepository is whether a recorded repository is the one a merge names, in either spelling it
|
||||||
|
// may have been recorded in: a path on the git seat, or the URL it was cloned from.
|
||||||
|
func sameRepository(repository string, m link.SourceMoved) bool {
|
||||||
|
want := strings.ToLower(m.Owner + "/" + m.Repo)
|
||||||
|
repo := strings.ToLower(strings.TrimSuffix(repository, ".git"))
|
||||||
|
return repo == want || strings.HasSuffix(repo, "/"+want) ||
|
||||||
|
(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
|
||||||
|
}
|
||||||
|
|
||||||
|
// readsFrom is whether a module's build read the repository a merge names: the second repository its
|
||||||
|
// recipe packages source from. Its ref must be the branch that moved, or unset — the same rule a
|
||||||
|
// module's own source follows.
|
||||||
|
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||||
|
for _, r := range read {
|
||||||
|
if sameRepository(r.Repository, m) && (r.Ref == "" || r.Ref == m.Base) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastLookAt is the most recent look at this repository by anything built from it.
|
||||||
|
func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
|
||||||
|
var newest time.Time
|
||||||
|
for _, e := range entries {
|
||||||
|
if sameRepository(e.Source.Repository, m) && e.Source.Seen.After(newest) {
|
||||||
|
newest = e.Source.Seen
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return newest
|
||||||
|
}
|
||||||
|
|
||||||
|
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed.
|
||||||
|
//
|
||||||
|
// **A change inside no module's own directory is a change to what they share.** The forge lists the
|
||||||
|
// files a merge changed; a module is affected when one of them is inside its own directory, when it
|
||||||
|
// is built from the repository's root — everything there is its source — or when some changed file
|
||||||
|
// belongs to no module's directory at all, which is how a shared file, a build recipe or a
|
||||||
|
// dependency at the root rebuilds everything built from that repository.
|
||||||
|
//
|
||||||
|
// A change inside *another* module's directory is that module's business and not this one's, even
|
||||||
|
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
|
||||||
|
// whatever branch it was registered from.
|
||||||
|
//
|
||||||
|
// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a
|
||||||
|
// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a
|
||||||
|
// change to shared code, because `modules/newmod` was nobody's known directory, and every module
|
||||||
|
// built from the repository was rebuilt and rolled out for a module none of them is. So the
|
||||||
|
// directories that hold modules are known too: the parents of the known modules' directories
|
||||||
|
// (`modules`, never the root). A changed path `<parent>/<name>/…` belongs to the module at
|
||||||
|
// `<parent>/<name>` — held or not — and rebuilds nothing else, **provided it is shown to be a
|
||||||
|
// module**: its `module.json` is among the changed files (added, changed, or removed with it). A
|
||||||
|
// directory under the same parent whose manifest the merge did not touch may as well be a shared
|
||||||
|
// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe
|
||||||
|
// direction: the fault this whole path exists for is a mesh that believes it is current and is not
|
||||||
|
// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was.
|
||||||
|
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
||||||
|
// Nothing said about the files, or not all of them said: everything built from it is affected.
|
||||||
|
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
var dirs []string
|
||||||
|
parents := map[string]bool{}
|
||||||
|
for _, e := range known {
|
||||||
|
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
|
||||||
|
dir := strings.Trim(e.Source.Path, "/")
|
||||||
|
dirs = append(dirs, dir)
|
||||||
|
if parent := path.Dir(dir); parent != "." && parent != "/" {
|
||||||
|
parents[parent] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
changed := map[string]bool{}
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
changed[strings.TrimPrefix(p, "/")] = true
|
||||||
|
}
|
||||||
|
for _, p := range m.Paths {
|
||||||
|
if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
var out []inventory.Entry
|
||||||
|
for _, e := range candidates {
|
||||||
|
if e.Source.Path == "" || anyInside(m.Paths, e.Source.Path) {
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know —
|
||||||
|
// `<parent>/<name>/…` under a directory known to hold modules, whose `module.json` the same merge
|
||||||
|
// changed (novox/hq issue 252). Such a file is that module's business and nobody else's.
|
||||||
|
func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool {
|
||||||
|
p = strings.TrimPrefix(p, "/")
|
||||||
|
for parent := range parents {
|
||||||
|
rest, under := strings.CutPrefix(p, parent+"/")
|
||||||
|
if !under {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, _, inADirectory := strings.Cut(rest, "/")
|
||||||
|
if !inADirectory || name == "" {
|
||||||
|
// A file directly in the parent — `modules/README.md` — is about all of them.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if changed[parent+"/"+name+"/module.json"] {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// inside is whether a changed file is in a directory: that directory itself, or under it.
|
||||||
|
func inside(path, dir string) bool {
|
||||||
|
dir = strings.Trim(dir, "/")
|
||||||
|
path = strings.TrimPrefix(path, "/")
|
||||||
|
return path == dir || strings.HasPrefix(path, dir+"/")
|
||||||
|
}
|
||||||
|
|
||||||
|
// insideAny is whether a changed file is in any of these directories.
|
||||||
|
func insideAny(path string, dirs []string) bool {
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if inside(path, dir) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// anyInside is whether any of these changed files is in a directory.
|
||||||
|
func anyInside(paths []string, dir string) bool {
|
||||||
|
for _, p := range paths {
|
||||||
|
if inside(p, dir) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// orderByBases is the entries with every base before what stands on it: a module whose build stood
|
||||||
|
// on another's artifact comes after that module. Entries outside the set are not waited for — they
|
||||||
|
// are not being rebuilt. Stable for what has no order between it.
|
||||||
|
//
|
||||||
|
// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are
|
||||||
|
// derived from builds, not declared, because a recorded manifest no longer carries `build.on`.
|
||||||
|
func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||||
|
inSet := map[string]bool{}
|
||||||
|
for _, e := range entries {
|
||||||
|
inSet[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
|
var out []inventory.Entry
|
||||||
|
placed := map[string]bool{}
|
||||||
|
var place func(e inventory.Entry, seen map[string]bool)
|
||||||
|
place = func(e inventory.Entry, seen map[string]bool) {
|
||||||
|
name := e.Manifest.Module
|
||||||
|
if placed[name] || seen[name] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[name] = true
|
||||||
|
for _, base := range entries {
|
||||||
|
if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) {
|
||||||
|
place(base, seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
placed[name] = true
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
place(e, map[string]bool{})
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||||
|
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||||
|
for _, e := range entries {
|
||||||
|
if standsOnModule(e, module, against) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
|
||||||
|
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
|
||||||
|
// — for a module registered from a manifest and not yet built — by the base its manifest names.
|
||||||
|
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
|
||||||
|
if e.Manifest.Module == module {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if e.Manifest.Build != nil {
|
||||||
|
for _, on := range e.Manifest.Build.On {
|
||||||
|
if on.Module == module {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
prefix := catalogue.ArtifactStoreScheme + module + "/"
|
||||||
|
for _, ref := range against[e.Manifest.Module] {
|
||||||
|
if strings.HasPrefix(ref, prefix) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
|
||||||
|
// with no time on it is taken as news: refusing it would silence a forge that says less.
|
||||||
|
func isHistory(mergedAt string, seen time.Time) bool {
|
||||||
|
if mergedAt == "" || seen.IsZero() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
at, err := time.Parse(time.RFC3339, mergedAt)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return at.Before(seen)
|
||||||
|
}
|
||||||
|
|
||||||
|
// dependentsOf is every catalogued module that stands on one of the moved modules, directly or
|
||||||
|
// through another dependent, and is not itself among them — in the catalogue's order, so the
|
||||||
|
// answer is the same each time. A module standing on nothing that moved is left alone: a merge
|
||||||
|
// rebuilds what it changed and what is built on top of that, not the catalogue.
|
||||||
|
func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||||
|
bases := map[string]bool{}
|
||||||
|
for _, e := range moved {
|
||||||
|
bases[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
|
var out []inventory.Entry
|
||||||
|
taken := map[string]bool{}
|
||||||
|
for grew := true; grew; {
|
||||||
|
grew = false
|
||||||
|
for _, e := range entries {
|
||||||
|
name := e.Manifest.Module
|
||||||
|
if bases[name] || taken[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for base := range bases {
|
||||||
|
if standsOnModule(e, base, against) {
|
||||||
|
taken[name] = true
|
||||||
|
out = append(out, e)
|
||||||
|
grew = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, e := range out {
|
||||||
|
bases[e.Manifest.Module] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// "4 machine(s), all doing what they were told, all heard from, running what the mesh would send
|
||||||
|
// them, and every module current with its source" was true for eleven hours of a mesh in which no
|
||||||
|
// module could reach another (novox/hq 04-ISSUES/145). Every question it answers is about the mesh
|
||||||
|
// and a machine agreeing; none of them dials anything.
|
||||||
|
|
||||||
|
// printed captures what a function writes to stdout.
|
||||||
|
func printed(t *testing.T, f func() error) string {
|
||||||
|
t.Helper()
|
||||||
|
old := os.Stdout
|
||||||
|
r, w, err := os.Pipe()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
os.Stdout = w
|
||||||
|
runErr := f()
|
||||||
|
_ = w.Close()
|
||||||
|
os.Stdout = old
|
||||||
|
var buf bytes.Buffer
|
||||||
|
if _, err := io.Copy(&buf, r); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if runErr != nil {
|
||||||
|
t.Fatal(runErr)
|
||||||
|
}
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheAllWellSentenceSaysWhatItDoesNotCover(t *testing.T) {
|
||||||
|
// A mesh with nothing to say. The sentence below was true of a mesh in which no module could
|
||||||
|
// reach another, for eleven hours.
|
||||||
|
got := printed(t, func() error {
|
||||||
|
return printStatus(answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}})
|
||||||
|
})
|
||||||
|
if !strings.Contains(got, "all doing what they were told") {
|
||||||
|
t.Fatalf("a mesh with nothing to say did not print the all-well sentence:\n%s", got)
|
||||||
|
}
|
||||||
|
// And now says what it is not a claim about.
|
||||||
|
for _, want := range []string{"Nothing here dials a provision", "04-ISSUES/145"} {
|
||||||
|
if !strings.Contains(got, want) {
|
||||||
|
t.Fatalf("the all-well sentence does not say %q:\n%s", want, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMeshWithSomethingToSayDoesNotPrintTheScopeLine(t *testing.T) {
|
||||||
|
// The scope belongs to the all-well sentence. A mesh with something wrong has specific things to
|
||||||
|
// read, and appending a caveat to those is noise.
|
||||||
|
got := printed(t, func() error {
|
||||||
|
return printStatus(answers{
|
||||||
|
nodes: []inventory.Node{{Name: "anchor"}},
|
||||||
|
untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}},
|
||||||
|
})
|
||||||
|
})
|
||||||
|
if strings.Contains(got, "Nothing here dials a provision") {
|
||||||
|
t.Fatalf("a mesh with a held module printed the all-well scope line:\n%s", got)
|
||||||
|
}
|
||||||
|
if strings.Contains(got, "all doing what they were told") {
|
||||||
|
t.Fatalf("a mesh with a held module printed the all-well sentence:\n%s", got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(got, "route-proxy") {
|
||||||
|
t.Fatalf("the held module is not named:\n%s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206).
|
||||||
|
// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build
|
||||||
|
// machine could not bind the worker the old controller had defined, and nothing could build the
|
||||||
|
// controller that would have redefined it. The built-by edge from the controller to its build
|
||||||
|
// machine yields to that order: the controller is built by whichever build machine is running.
|
||||||
|
func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) {
|
||||||
|
edges := []inventory.Edge{
|
||||||
|
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||||
|
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||||
|
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||||
|
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||||
|
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||||
|
}
|
||||||
|
set := reachableFrom([]string{"mesh-controller"}, edges)
|
||||||
|
if len(set) != 3 {
|
||||||
|
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
|
||||||
|
}
|
||||||
|
tiers := tiersOf(set, edges)
|
||||||
|
pos := map[string]int{}
|
||||||
|
for i, tier := range tiers {
|
||||||
|
for _, m := range tier {
|
||||||
|
pos[m] = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if pos["mesh-controller"] != 0 {
|
||||||
|
t.Fatalf("the controller first, built by the build machine that is running: %v", tiers)
|
||||||
|
}
|
||||||
|
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||||
|
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||||
|
}
|
||||||
|
if pos["route-proxy"] <= pos["build-agent"] {
|
||||||
|
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
||||||
|
}
|
||||||
|
if hasCycle(tiers, edges) {
|
||||||
|
t.Fatalf("no cycle here: %v", tiers)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
# examples
|
||||||
|
|
||||||
|
Things that run, kept here because a contract is easier to read as working code than as prose.
|
||||||
|
|
||||||
|
**Nothing here is part of the control plane.** The control plane decides and never touches a
|
||||||
|
machine ([README](../README.md)); everything in this directory runs *on* a machine and touches it.
|
||||||
|
These are reference implementations of contracts the control plane defines, and a real one ships
|
||||||
|
with the module that ships the software it configures.
|
||||||
|
|
||||||
|
| | |
|
||||||
|
|---|---|
|
||||||
|
| `postgres-provisioner` | the last step of a credential: reads what the mesh delivered and makes PostgreSQL accept it |
|
||||||
|
|
||||||
|
## Running the provisioner
|
||||||
|
|
||||||
|
`--watch` reconciles now and again whenever what the mesh delivered changes. That is what lets it
|
||||||
|
be a module: an ordinary long-running service the host supervises, rather than something that has
|
||||||
|
to be invoked after every declaration by a timer or a unit wired to a file.
|
||||||
|
|
||||||
|
It polls rather than watching the filesystem, because the host writes atomically — the file is
|
||||||
|
replaced, so a watch on the path stops seeing anything after the first replacement. A watcher that
|
||||||
|
silently stops working is worse than a poll.
|
||||||
|
|
||||||
|
Credentials are compared by digest and never by content. This runs for as long as the machine is
|
||||||
|
up, and a secret does not belong in a long-lived variable when a hash answers the same question.
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
# The bucket provisioner, as a module ships one.
|
||||||
|
#
|
||||||
|
# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on
|
||||||
|
# it.
|
||||||
|
#
|
||||||
|
# **Not FROM scratch, unlike the postgres one, and the difference is the point.** This drives the
|
||||||
|
# store's own command line, so that client has to be in the image — a provisioner is allowed to
|
||||||
|
# know how to operate the thing it provisions.
|
||||||
|
#
|
||||||
|
# The client is copied from the vendor's own image rather than installed from a distribution:
|
||||||
|
# `apk add mc` on Alpine installs Midnight Commander, which is a different program with the same
|
||||||
|
# name, and the failure would be a provisioner that starts cleanly and cannot do anything.
|
||||||
|
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||||
|
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||||
|
ARG GO_BASE=golang:1.26-alpine
|
||||||
|
FROM ${GO_BASE} AS build
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \
|
||||||
|
-o /mesh-provision-objectstore ./examples/objectstore-provisioner
|
||||||
|
|
||||||
|
# Pinned like everything else the mesh runs (novox/hq ADR 0006): a tag moves and a digest does not.
|
||||||
|
FROM minio/mc:latest AS client
|
||||||
|
|
||||||
|
FROM alpine:3.21
|
||||||
|
RUN apk add --no-cache ca-certificates
|
||||||
|
COPY --from=client /usr/bin/mc /usr/bin/mc
|
||||||
|
COPY --from=build /mesh-provision-objectstore /mesh-provision-objectstore
|
||||||
|
# Watching by default, because that is what makes it a module: an ordinary long-running service
|
||||||
|
# the host supervises, rather than something invoked after every declaration.
|
||||||
|
ENTRYPOINT ["/mesh-provision-objectstore", "--watch"]
|
||||||
@@ -0,0 +1,384 @@
|
|||||||
|
// A provisioner for buckets, in the form the mesh expects one.
|
||||||
|
//
|
||||||
|
// The same contract as `examples/postgres-provisioner`, against a different kind of thing — and
|
||||||
|
// that is the point of it existing. The mesh generated a secret, sealed it to the machine that
|
||||||
|
// must accept it, and discarded the plaintext, so it cannot tell an object store to start
|
||||||
|
// accepting it. Something on that machine reads what the host wrote and makes it true.
|
||||||
|
//
|
||||||
|
// **It is an example, not part of the control plane** (novox/hq ADR 0001). A real one ships with
|
||||||
|
// the module that ships the object store. What lives here is the contract, written as something
|
||||||
|
// that runs so it can be read rather than described.
|
||||||
|
//
|
||||||
|
// What it is given, both written by the host from an ordinary declaration:
|
||||||
|
//
|
||||||
|
// $GRANTS/mesh.json every consumer, what it asked for, and where its credential is
|
||||||
|
// $GRANTS/<node>.secret one consumer's secret key, alone in the file
|
||||||
|
//
|
||||||
|
// **Why it drives the vendor's own command line rather than an SDK.** MinIO's admin API encrypts
|
||||||
|
// its request bodies, which is why a separate admin library exists; pulling that in would add a
|
||||||
|
// system-metrics dependency tree to a repository that has none, to create a user. A provisioner
|
||||||
|
// is allowed to know how to operate the thing it provisions — that is the whole of its job — and
|
||||||
|
// this way the example stays about the contract instead of about somebody's request signing.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"os/signal"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mark is what this provisioner names the access keys it owns.
|
||||||
|
//
|
||||||
|
// So it never removes one a person made by hand — the mesh's own rule about origins, one level
|
||||||
|
// down (novox/hq 04-ISSUES/010). A provisioner that deleted every user it did not recognise would
|
||||||
|
// be one nobody could safely run against a store that predates it.
|
||||||
|
const mark = "mesh_"
|
||||||
|
|
||||||
|
// alias is the name `mc` keeps its connection under. Local to this process's config directory.
|
||||||
|
const alias = "store"
|
||||||
|
|
||||||
|
// contribution is one consumer, as the mesh described it.
|
||||||
|
type contribution struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
// As is what to call the login this consumer will use.
|
||||||
|
//
|
||||||
|
// **Given, not invented** (novox/hq 04-ISSUES/023). This provisioner used to make the name
|
||||||
|
// itself, which worked and meant the consumer — the one thing that has to present it — could
|
||||||
|
// not learn it. The mesh derives it once and sends it to both ends.
|
||||||
|
As string `json:"as"`
|
||||||
|
Secret string `json:"secret"`
|
||||||
|
Values map[string]any `json:"values"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifest struct {
|
||||||
|
Requirement string `json:"requirement"`
|
||||||
|
Given []contribution `json:"given"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
if len(os.Args) > 1 && os.Args[1] == "--watch" {
|
||||||
|
if err := watch(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-provision-objectstore: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := run(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-provision-objectstore: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watch reconciles now, and again whenever what the mesh delivered changes.
|
||||||
|
//
|
||||||
|
// By polling rather than watching the filesystem: the host writes atomically, so the file is
|
||||||
|
// replaced rather than modified, and an inotify watch on the path stops seeing anything after the
|
||||||
|
// first replacement — a watcher that silently stops working.
|
||||||
|
func watch(ctx context.Context) error {
|
||||||
|
const every = 10 * time.Second
|
||||||
|
var last string
|
||||||
|
|
||||||
|
for {
|
||||||
|
state, err := given()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
// Said and retried. A provisioner that exits because the mesh has not written
|
||||||
|
// anything yet has to be restarted by hand after the first push.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err)
|
||||||
|
case state != last:
|
||||||
|
if err := run(ctx); err != nil {
|
||||||
|
// Reported and retried. The usual reason is that the store has not finished
|
||||||
|
// starting, and giving up would mean a module that works only when two
|
||||||
|
// containers happen to come up in the right order.
|
||||||
|
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||||
|
} else {
|
||||||
|
last = state
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case <-time.After(every):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantsDir is where the host was told to write what this provisioner is given.
|
||||||
|
func grantsDir() string {
|
||||||
|
if d := os.Getenv("GRANTS"); d != "" {
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
return "/var/lib/objectstore/grants"
|
||||||
|
}
|
||||||
|
|
||||||
|
// given is everything the mesh has delivered, as one string, so a change to any of it is one
|
||||||
|
// comparison.
|
||||||
|
//
|
||||||
|
// Credentials are included by their **digest**, never their content: this is compared, held in
|
||||||
|
// memory for the life of the process, and printed on failure, and a secret belongs in none of
|
||||||
|
// those when a hash answers the same question.
|
||||||
|
func given() (string, error) {
|
||||||
|
entries, err := os.ReadDir(grantsDir())
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, e := range entries {
|
||||||
|
names = append(names, e.Name())
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
sum := sha256.New()
|
||||||
|
for _, name := range names {
|
||||||
|
body, err := os.ReadFile(filepath.Join(grantsDir(), name))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body))
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(sum.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(ctx context.Context) error {
|
||||||
|
raw, err := os.ReadFile(filepath.Join(grantsDir(), "mesh.json"))
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// Nothing has been granted here. Not a failure: a provider with no consumers is an
|
||||||
|
// ordinary state, and one this must be able to reach from any other.
|
||||||
|
fmt.Printf("nothing has been granted to this machine\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var m manifest
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
return fmt.Errorf("the manifest at %s is not readable: %w", grantsDir(), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := connect(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// **Reconciling, not applying a change.** It runs after every declaration and is never told
|
||||||
|
// what changed, so it must reach the same state from wherever it starts.
|
||||||
|
wanted := map[string]bool{}
|
||||||
|
for _, c := range sorted(m.Given) {
|
||||||
|
if c.Node == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
bucket, _ := c.Values["bucket"].(string)
|
||||||
|
if bucket == "" {
|
||||||
|
return fmt.Errorf("%s asked for a bucket and did not name it", c.Node)
|
||||||
|
}
|
||||||
|
if err := usableBucketName(bucket); err != nil {
|
||||||
|
return fmt.Errorf("%s asked for a bucket named %q: %w", c.Node, bucket, err)
|
||||||
|
}
|
||||||
|
secret, err := os.ReadFile(c.Secret)
|
||||||
|
if err != nil {
|
||||||
|
// The manifest says there is a credential and the host has not written it. Refused
|
||||||
|
// rather than creating a user with no secret — a login nothing can use, which
|
||||||
|
// nothing would report until something tried to connect.
|
||||||
|
return fmt.Errorf("%s's credential should be at %s and is not there", c.Node, c.Secret)
|
||||||
|
}
|
||||||
|
|
||||||
|
// One access key per consumer, and a consumer is a module on a machine (novox/hq
|
||||||
|
// 04-ISSUES/022) — otherwise every service on a node shares one key, and the policy that
|
||||||
|
// confines each to its own bucket confines none of them.
|
||||||
|
//
|
||||||
|
// The name comes from the mesh (04-ISSUES/023), so the consumer knows what it will be.
|
||||||
|
key := c.As
|
||||||
|
if key == "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s on %s was granted a bucket and the mesh did not say what to call its key, "+
|
||||||
|
"so there is no name both ends would agree on", c.From, c.Node)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(key, mark) {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s on %s is to be called %q, which does not begin with %q — this provisioner "+
|
||||||
|
"removes what it made by that prefix, so it would never let this one go",
|
||||||
|
c.From, c.Node, key, mark)
|
||||||
|
}
|
||||||
|
wanted[key] = true
|
||||||
|
if err := ensureBucket(ctx, bucket); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ensureUser(ctx, key, strings.TrimSpace(string(secret))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ensureOnlyThatBucket(ctx, key, bucket); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And everything this provisioner made that nobody asks for any more. **The half usually
|
||||||
|
// missing**: a consumer that goes away otherwise keeps a working key for ever, and nothing
|
||||||
|
// says so.
|
||||||
|
return revokeOrphans(ctx, wanted)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sorted puts the consumers in a stable order, so two runs over the same input do the same things
|
||||||
|
// in the same sequence and a log can be compared against another.
|
||||||
|
func sorted(given []contribution) []contribution {
|
||||||
|
out := append([]contribution(nil), given...)
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].Node != out[j].Node {
|
||||||
|
return out[i].Node < out[j].Node
|
||||||
|
}
|
||||||
|
return out[i].From < out[j].From
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// usableBucketName refuses a name the store would refuse, but says so in terms of the module that
|
||||||
|
// asked rather than in terms of a REST error nobody can trace back.
|
||||||
|
//
|
||||||
|
// Deliberately narrower than the store's own rule: this rejects what is ambiguous as well as what
|
||||||
|
// is invalid, because a bucket named `Photos` that arrives as `photos` is a module that works
|
||||||
|
// until somebody looks.
|
||||||
|
func usableBucketName(name string) error {
|
||||||
|
if len(name) < 3 || len(name) > 63 {
|
||||||
|
return fmt.Errorf("a bucket name is 3 to 63 characters")
|
||||||
|
}
|
||||||
|
for _, r := range name {
|
||||||
|
if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' {
|
||||||
|
return fmt.Errorf("only lower-case letters, digits and dashes are usable, and %q is not", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if name[0] == '-' || name[len(name)-1] == '-' {
|
||||||
|
return fmt.Errorf("it may not begin or end with a dash")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// onlyThatBucket is the policy a consumer gets: its own bucket, and nothing else in the store.
|
||||||
|
//
|
||||||
|
// **Written per consumer rather than shared.** A single policy naming every bucket would grow a
|
||||||
|
// line each time somebody asks for one, and every existing consumer would silently gain access to
|
||||||
|
// the new one.
|
||||||
|
func onlyThatBucket(bucket string) string {
|
||||||
|
return fmt.Sprintf(`{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{"Effect": "Allow",
|
||||||
|
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s"]},
|
||||||
|
{"Effect": "Allow",
|
||||||
|
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||||
|
"Resource": ["arn:aws:s3:::%s/*"]}
|
||||||
|
]
|
||||||
|
}`, bucket, bucket)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mc(ctx context.Context, args ...string) (string, error) {
|
||||||
|
// Its own configuration directory, so this never reads or writes whatever a person running
|
||||||
|
// `mc` on this machine has set up.
|
||||||
|
cmd := exec.CommandContext(ctx, "mc", append([]string{"--config-dir", "/tmp/mesh-mc"}, args...)...)
|
||||||
|
out, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
// The command is named without its arguments: an argument here can be a secret.
|
||||||
|
return string(out), fmt.Errorf("mc %s: %w\n%s", args[0], err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return string(out), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// connect points `mc` at the store, using the root credential this provisioner was given.
|
||||||
|
func connect(ctx context.Context) error {
|
||||||
|
endpoint := os.Getenv("MESH_OBJECTSTORE_URL")
|
||||||
|
if endpoint == "" {
|
||||||
|
endpoint = "http://127.0.0.1:9000"
|
||||||
|
}
|
||||||
|
user := os.Getenv("MESH_OBJECTSTORE_ROOT_USER")
|
||||||
|
file := os.Getenv("MESH_OBJECTSTORE_ROOT_PASSWORD_FILE")
|
||||||
|
if user == "" || file == "" {
|
||||||
|
// Named together, because a provisioner that starts with half its credential fails later
|
||||||
|
// against the store and reads as the store being wrong.
|
||||||
|
return fmt.Errorf("MESH_OBJECTSTORE_ROOT_USER and MESH_OBJECTSTORE_ROOT_PASSWORD_FILE " +
|
||||||
|
"must both be set: this provisions the store and has to authenticate to it")
|
||||||
|
}
|
||||||
|
// From a file, never an environment variable: the environment of a process is readable by
|
||||||
|
// anything that can see the process, and this one is the store's root.
|
||||||
|
password, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the store's root password should be at %s and is not there", file)
|
||||||
|
}
|
||||||
|
_, err = mc(ctx, "alias", "set", alias, endpoint, user, strings.TrimSpace(string(password)))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureBucket(ctx context.Context, bucket string) error {
|
||||||
|
_, err := mc(ctx, "mb", "--ignore-existing", alias+"/"+bucket)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ensureUser sets the secret every time rather than only on creation.
|
||||||
|
//
|
||||||
|
// The mesh replaces the file when it rotates, and a provisioner that only ever created would
|
||||||
|
// leave the old key working — a rotation that reports success and changes nothing.
|
||||||
|
func ensureUser(ctx context.Context, key, secret string) error {
|
||||||
|
_, err := mc(ctx, "admin", "user", "add", alias, key, secret)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureOnlyThatBucket(ctx context.Context, key, bucket string) error {
|
||||||
|
name := key + "-" + bucket
|
||||||
|
path := filepath.Join("/tmp", name+".json")
|
||||||
|
if err := os.WriteFile(path, []byte(onlyThatBucket(bucket)), 0o600); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer os.Remove(path)
|
||||||
|
|
||||||
|
// Replaced rather than created-if-absent: the bucket a consumer asks for can change, and a
|
||||||
|
// policy that was only ever created would keep granting the old one as well.
|
||||||
|
_, _ = mc(ctx, "admin", "policy", "remove", alias, name)
|
||||||
|
if _, err := mc(ctx, "admin", "policy", "create", alias, name, path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := mc(ctx, "admin", "policy", "attach", alias, name, "--user", key)
|
||||||
|
if err != nil && strings.Contains(err.Error(), "already") {
|
||||||
|
// Attaching a policy that is already attached is the state being asked for.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// revokeOrphans removes the keys this provisioner made that nobody asks for any more.
|
||||||
|
func revokeOrphans(ctx context.Context, wanted map[string]bool) error {
|
||||||
|
out, err := mc(ctx, "admin", "user", "list", alias, "--json")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var u struct {
|
||||||
|
AccessKey string `json:"accessKey"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal([]byte(line), &u) != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(u.AccessKey, mark) || wanted[u.AccessKey] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := mc(ctx, "admin", "user", "remove", alias, u.AccessKey); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("revoked %s, which nothing asks for any more\n", u.AccessKey)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
// A bucket name is checked here so the refusal names the module that asked.
|
||||||
|
//
|
||||||
|
// The store would refuse most of these itself, as a REST error arriving inside a provisioner log,
|
||||||
|
// with nothing saying which consumer's manifest caused it.
|
||||||
|
func TestABucketNameThatWouldNotWorkIsRefusedHere(t *testing.T) {
|
||||||
|
for _, name := range []string{
|
||||||
|
"", // nothing asked for
|
||||||
|
"ab", // too short
|
||||||
|
"-lead",
|
||||||
|
"trail-",
|
||||||
|
"Photos", // upper case: arrives lower-cased, and works until somebody looks
|
||||||
|
"my_bucket", // underscore
|
||||||
|
"a.b", // dots are legal in S3 and break TLS host matching; not worth the surprise
|
||||||
|
} {
|
||||||
|
if err := usableBucketName(name); err == nil {
|
||||||
|
t.Errorf("%q was accepted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnOrdinaryBucketNameIsAccepted(t *testing.T) {
|
||||||
|
for _, name := range []string{"photos", "a-b-c", "backups2026", "abc"} {
|
||||||
|
if err := usableBucketName(name); err != nil {
|
||||||
|
t.Errorf("%q was refused: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The policy a consumer gets names its own bucket and nothing else.
|
||||||
|
//
|
||||||
|
// **The half that matters is what it does not say.** A policy granting `arn:aws:s3:::*` would
|
||||||
|
// pass every test that checks a consumer can reach its own bucket, and would give every consumer
|
||||||
|
// the whole store.
|
||||||
|
func TestThePolicyGrantsOneBucketAndNoOther(t *testing.T) {
|
||||||
|
policy := onlyThatBucket("photos")
|
||||||
|
for _, want := range []string{`"arn:aws:s3:::photos"`, `"arn:aws:s3:::photos/*"`} {
|
||||||
|
if !contains(policy, want) {
|
||||||
|
t.Errorf("the policy does not carry %s:\n%s", want, policy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, unwanted := range []string{`:::*`, `"*"`, `:::photos-other`} {
|
||||||
|
if contains(policy, unwanted) {
|
||||||
|
t.Errorf("the policy carries %s, which reaches beyond the bucket asked for:\n%s",
|
||||||
|
unwanted, policy)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A policy is per consumer, so one asking for a second bucket cannot widen another's.
|
||||||
|
func TestTwoConsumersGetPoliciesThatDoNotOverlap(t *testing.T) {
|
||||||
|
if contains(onlyThatBucket("photos"), "invoices") ||
|
||||||
|
contains(onlyThatBucket("invoices"), "photos") {
|
||||||
|
t.Error("a consumer's policy names another consumer's bucket")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(haystack, needle string) bool {
|
||||||
|
for i := 0; i+len(needle) <= len(haystack); i++ {
|
||||||
|
if haystack[i:i+len(needle)] == needle {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# The provisioner, as a module ships one.
|
||||||
|
#
|
||||||
|
# Built here so a machine can be given it by the mesh rather than by somebody putting a binary on
|
||||||
|
# it. Static and FROM scratch for the same reason the control plane's image is: it is fetched by
|
||||||
|
# digest and run on a machine, and everything in it is something a person would have to audit.
|
||||||
|
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||||
|
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||||
|
ARG GO_BASE=golang:1.26-alpine
|
||||||
|
FROM ${GO_BASE} AS build
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \
|
||||||
|
-o /mesh-provision-postgres ./examples/postgres-provisioner
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
COPY --from=build /mesh-provision-postgres /mesh-provision-postgres
|
||||||
|
# Watching by default, because that is what makes it a module: an ordinary long-running service
|
||||||
|
# the host supervises, rather than something invoked after every declaration.
|
||||||
|
ENTRYPOINT ["/mesh-provision-postgres", "--watch"]
|
||||||
@@ -0,0 +1,415 @@
|
|||||||
|
// A provisioner, in the form the mesh expects one.
|
||||||
|
//
|
||||||
|
// The mesh generated a password, sealed it to the machine that must accept it, and discarded the
|
||||||
|
// plaintext — so it cannot tell PostgreSQL to start accepting it. Something on that machine reads
|
||||||
|
// what the host wrote and makes it true. This is that something.
|
||||||
|
//
|
||||||
|
// **It is an example, not part of the control plane.** The control plane decides and never
|
||||||
|
// touches a machine; this runs on the machine and touches it. A real one ships with the module
|
||||||
|
// that ships PostgreSQL (novox/hq ADR 0001 — third-party software runs *on* the mesh, not *of*
|
||||||
|
// it). What lives here is the contract, written as something that runs so it can be read rather
|
||||||
|
// than described.
|
||||||
|
//
|
||||||
|
// What it is given, both written by the host from an ordinary declaration:
|
||||||
|
//
|
||||||
|
// $GRANTS/mesh.json every consumer, what it asked for, and where its credential is
|
||||||
|
// $GRANTS/<node>.secret one consumer's password, alone in the file
|
||||||
|
//
|
||||||
|
// Two files because the mesh discarded the value and could not compose a document containing it.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mark is what this provisioner names the roles it owns.
|
||||||
|
//
|
||||||
|
// So it never removes one a person made by hand — the mesh's own rule about origins, one level
|
||||||
|
// down (novox/hq 04-ISSUES/010). A provisioner that dropped every role it did not recognise would
|
||||||
|
// be a provisioner nobody could safely run on a database that predates it.
|
||||||
|
const mark = "mesh_"
|
||||||
|
|
||||||
|
// contribution is one consumer, as the mesh described it.
|
||||||
|
type contribution struct {
|
||||||
|
From string `json:"from"`
|
||||||
|
Node string `json:"node"`
|
||||||
|
// As is what to call the login this consumer will use.
|
||||||
|
//
|
||||||
|
// **Given, not invented** (novox/hq 04-ISSUES/023). This provisioner used to make the name
|
||||||
|
// itself, which worked and meant the consumer — the one thing that has to present it — could
|
||||||
|
// not learn it. The mesh derives it once and sends it to both ends.
|
||||||
|
As string `json:"as"`
|
||||||
|
Secret string `json:"secret"`
|
||||||
|
Values map[string]any `json:"values"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type manifest struct {
|
||||||
|
Requirement string `json:"requirement"`
|
||||||
|
Given []contribution `json:"given"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// Once, or whenever what it was given changes.
|
||||||
|
//
|
||||||
|
// **Watching is what lets this be a module.** Run once, it has to be invoked by something
|
||||||
|
// after every declaration — a timer that runs it when nothing changed, or a unit wired to
|
||||||
|
// restart on a file. Watching, it is an ordinary long-running service, which is a shape the
|
||||||
|
// mesh already delivers and the host already supervises.
|
||||||
|
//
|
||||||
|
// The file it watches is the manifest the mesh writes. A credential changing rewrites the
|
||||||
|
// file beside it and not the manifest, so the manifest is stamped whenever either is written
|
||||||
|
// — which is why this compares content rather than modification time.
|
||||||
|
if len(os.Args) > 1 && os.Args[1] == "--watch" {
|
||||||
|
if err := watch(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := run(ctx); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-provision-postgres: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// watch reconciles now, and again whenever what the mesh delivered changes.
|
||||||
|
//
|
||||||
|
// By polling rather than by watching the filesystem, because the file is replaced rather than
|
||||||
|
// written in place — the host writes atomically, so an inotify watch on the path stops seeing
|
||||||
|
// anything after the first replacement, which is a watcher that silently stops working.
|
||||||
|
func watch(ctx context.Context) error {
|
||||||
|
const every = 10 * time.Second
|
||||||
|
var last string
|
||||||
|
|
||||||
|
for {
|
||||||
|
state, err := given()
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
// Said and retried. A provisioner that exits because the mesh has not written
|
||||||
|
// anything yet is one that has to be restarted by hand after the first push.
|
||||||
|
fmt.Fprintf(os.Stderr, "cannot read what was granted: %v\n", err)
|
||||||
|
case state != last:
|
||||||
|
if err := run(ctx); err != nil {
|
||||||
|
// Reported and retried. The usual reason is that the database has not finished
|
||||||
|
// starting, and giving up would mean a module that works only if the two
|
||||||
|
// containers happen to come up in the right order.
|
||||||
|
fmt.Fprintf(os.Stderr, "%v\n", err)
|
||||||
|
} else {
|
||||||
|
last = state
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case <-time.After(every):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// given is everything the mesh has delivered, as one string, so a change of any of it is one
|
||||||
|
// comparison.
|
||||||
|
//
|
||||||
|
// The credentials are included by their **digest**, never their content: this is compared, logged
|
||||||
|
// on nothing, and held in memory for as long as the process runs, and a secret does not belong in
|
||||||
|
// any of that when a hash answers the same question.
|
||||||
|
func given() (string, error) {
|
||||||
|
grants := os.Getenv("GRANTS")
|
||||||
|
if grants == "" {
|
||||||
|
grants = "/var/lib/postgres/grants"
|
||||||
|
}
|
||||||
|
entries, err := os.ReadDir(grants)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for _, e := range entries {
|
||||||
|
names = append(names, e.Name())
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
|
||||||
|
sum := sha256.New()
|
||||||
|
for _, name := range names {
|
||||||
|
body, err := os.ReadFile(filepath.Join(grants, name))
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
fmt.Fprintf(sum, "%s:%x\n", name, sha256.Sum256(body))
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(sum.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(ctx context.Context) error {
|
||||||
|
grants := os.Getenv("GRANTS")
|
||||||
|
if grants == "" {
|
||||||
|
grants = "/var/lib/postgres/grants"
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(filepath.Join(grants, "mesh.json"))
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
// Nothing has been granted here. Not a failure: a provider with no consumers is an
|
||||||
|
// ordinary state, and one this must be able to reach from any other.
|
||||||
|
fmt.Printf("nothing has been granted to this machine\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var m manifest
|
||||||
|
if err := json.Unmarshal(raw, &m); err != nil {
|
||||||
|
return fmt.Errorf("the manifest at %s is not readable: %w", grants, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
where, err := connectionString()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db, err := pgx.Connect(ctx, where)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer db.Close(ctx)
|
||||||
|
|
||||||
|
// **Reconciling, not applying a change.** It runs after every declaration and is never told
|
||||||
|
// what changed, so it must reach the same state from wherever it starts.
|
||||||
|
wanted := map[string]bool{}
|
||||||
|
for _, c := range sorted(m.Given) {
|
||||||
|
if c.Node == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, _ := c.Values["name"].(string)
|
||||||
|
if name == "" {
|
||||||
|
return fmt.Errorf("%s asked for a database and did not name it", c.Node)
|
||||||
|
}
|
||||||
|
password, err := os.ReadFile(c.Secret)
|
||||||
|
if err != nil {
|
||||||
|
// The manifest says there is a credential and the host has not written it. Refused
|
||||||
|
// rather than creating a role with no password — a login nothing can use, which
|
||||||
|
// nothing would report until something tried to connect.
|
||||||
|
return fmt.Errorf("%s's credential should be at %s and is not there", c.Node, c.Secret)
|
||||||
|
}
|
||||||
|
|
||||||
|
// **The name the mesh gave this consumer** — a module on a machine (novox/hq
|
||||||
|
// 04-ISSUES/022), said by the mesh rather than derived here (04-ISSUES/023).
|
||||||
|
//
|
||||||
|
// Refused rather than derived when it is absent. Falling back to a name of this
|
||||||
|
// provisioner's own would work, and the consumer would be told a different one and fail
|
||||||
|
// to authenticate against a role that exists — which is the worst of the three outcomes,
|
||||||
|
// because everything reports success.
|
||||||
|
role := c.As
|
||||||
|
if role == "" {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s on %s was granted a database and the mesh did not say what to call its "+
|
||||||
|
"login, so there is no name both ends would agree on", c.From, c.Node)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(role, mark) {
|
||||||
|
// Withdrawal finds this provisioner's work by prefix. A login without it would never
|
||||||
|
// be removed, and would keep working for ever after its consumer went away.
|
||||||
|
return fmt.Errorf(
|
||||||
|
"%s on %s is to be called %q, which does not begin with %q — this provisioner "+
|
||||||
|
"removes what it made by that prefix, so it would never let this one go",
|
||||||
|
c.From, c.Node, role, mark)
|
||||||
|
}
|
||||||
|
wanted[role] = true
|
||||||
|
if err := ensureRole(ctx, db, role, strings.TrimSpace(string(password))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := ensureDatabase(ctx, db, name, role); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And everything this provisioner made that nobody asks for any more. **The half usually
|
||||||
|
// missing**: a consumer that goes away otherwise keeps a working login for ever and nothing
|
||||||
|
// says so.
|
||||||
|
return revokeOrphans(ctx, db, wanted)
|
||||||
|
}
|
||||||
|
|
||||||
|
// identifierLimit is where PostgreSQL stops reading a name: NAMEDATALEN - 1.
|
||||||
|
const identifierLimit = 63
|
||||||
|
|
||||||
|
// usableRole refuses a role name PostgreSQL would silently shorten.
|
||||||
|
//
|
||||||
|
// **Truncation is a NOTICE, not an error.** A name past the limit is cut to fit and the statement
|
||||||
|
// succeeds, so two consumers whose names agree for the first 63 bytes become one role — which is
|
||||||
|
// the exact fault 022 was about, reappearing at a length nobody would think to test. Refusing is
|
||||||
|
// the only honest answer: the provisioner cannot shorten the name itself without inventing a
|
||||||
|
// second naming scheme that the mesh does not know about, and would then be creating a login the
|
||||||
|
// mesh cannot name.
|
||||||
|
func usableRole(role string) error {
|
||||||
|
if len(role) <= identifierLimit {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"the role for this consumer would be %q, which is %d bytes and PostgreSQL keeps %d — "+
|
||||||
|
"it would be shortened silently, and another consumer shortened to the same name "+
|
||||||
|
"would share the login. Shorten the node or module name",
|
||||||
|
role, len(role), identifierLimit)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureRole(ctx context.Context, db *pgx.Conn, role, password string) error {
|
||||||
|
if err := usableRole(role); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var exists bool
|
||||||
|
if err := db.QueryRow(ctx,
|
||||||
|
`select true from pg_roles where rolname = $1`, role).Scan(&exists); err != nil && err != pgx.ErrNoRows {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Set every time rather than only on creation. The mesh replaces the file when it rotates,
|
||||||
|
// and a provisioner that only ever created would leave the old password working — a rotation
|
||||||
|
// that reports success and changes nothing.
|
||||||
|
verb := "create"
|
||||||
|
if exists {
|
||||||
|
verb = "alter"
|
||||||
|
}
|
||||||
|
_, err := db.Exec(ctx, fmt.Sprintf("%s role %s with login password %s",
|
||||||
|
verb, quoteName(role), quoteString(password)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
fmt.Printf("created %s\n", role)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ensureDatabase(ctx context.Context, db *pgx.Conn, name, owner string) error {
|
||||||
|
var exists bool
|
||||||
|
if err := db.QueryRow(ctx,
|
||||||
|
`select true from pg_database where datname = $1`, name).Scan(&exists); err != nil && err != pgx.ErrNoRows {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if _, err := db.Exec(ctx, fmt.Sprintf("create database %s owner %s",
|
||||||
|
quoteName(name), quoteName(owner))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("created database %s owned by %s\n", name, owner)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func revokeOrphans(ctx context.Context, db *pgx.Conn, wanted map[string]bool) error {
|
||||||
|
rows, err := db.Query(ctx,
|
||||||
|
`select rolname from pg_roles where rolname like $1 and rolcanlogin order by rolname`,
|
||||||
|
mark+"%")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var found []string
|
||||||
|
for rows.Next() {
|
||||||
|
var role string
|
||||||
|
if err := rows.Scan(&role); err != nil {
|
||||||
|
rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
found = append(found, role)
|
||||||
|
}
|
||||||
|
rows.Close()
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, role := range found {
|
||||||
|
if wanted[role] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Login removed rather than the role dropped. Dropping fails while the role owns
|
||||||
|
// anything, and a provisioner that failed there would stop reconciling everything else —
|
||||||
|
// so the credential stops working immediately and what it owns is somebody's to decide
|
||||||
|
// about.
|
||||||
|
if _, err := db.Exec(ctx, fmt.Sprintf("alter role %s with nologin",
|
||||||
|
quoteName(role))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
fmt.Printf("revoked %s — nothing in the mesh asks for it\n", role)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sorted puts consumers in a stable order, so two runs do the same work in the same sequence and
|
||||||
|
// the output of one can be compared with another.
|
||||||
|
func sorted(given []contribution) []contribution {
|
||||||
|
out := append([]contribution{}, given...)
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].Node != out[j].Node {
|
||||||
|
return out[i].Node < out[j].Node
|
||||||
|
}
|
||||||
|
return out[i].From < out[j].From
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// quoteName and quoteString exist because PostgreSQL takes no parameters in DDL.
|
||||||
|
//
|
||||||
|
// Both double the quote character, which is the whole of the escaping rule. Worth doing properly
|
||||||
|
// even here: a password is chosen by the mesh and a node name by a person, and "the value happens
|
||||||
|
// to be safe today" is not a property anything should rest on.
|
||||||
|
func quoteName(s string) string { return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` }
|
||||||
|
func quoteString(s string) string { return `'` + strings.ReplaceAll(s, `'`, `''`) + `'` }
|
||||||
|
|
||||||
|
// connectionString is where this provisioner reaches the database it owns.
|
||||||
|
//
|
||||||
|
// **The password comes from a file**, because that is how the mesh delivers one. A module's own
|
||||||
|
// secret — a superuser password here — is sealed to the machine and written by the host; a
|
||||||
|
// provisioner told to take it from an environment variable would need somebody to read the file
|
||||||
|
// and pass it in, which is a person in the middle of the one path that exists so there is not
|
||||||
|
// one.
|
||||||
|
//
|
||||||
|
// It is also the difference between a credential that lives in a file and one that lives in a
|
||||||
|
// process listing: `docker inspect` prints environment, and a superuser password printed by an
|
||||||
|
// ordinary diagnostic is a superuser password in whatever collected that diagnostic.
|
||||||
|
//
|
||||||
|
// MESH_PROVISION_POSTGRES alone still works, for a provisioner somebody runs by hand.
|
||||||
|
func connectionString() (string, error) {
|
||||||
|
where := strings.TrimSpace(os.Getenv("MESH_PROVISION_POSTGRES"))
|
||||||
|
if where == "" {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"MESH_PROVISION_POSTGRES is not set, so this provisioner does not know which " +
|
||||||
|
"database it owns")
|
||||||
|
}
|
||||||
|
path := strings.TrimSpace(os.Getenv("MESH_PROVISION_PASSWORD_FILE"))
|
||||||
|
if path == "" {
|
||||||
|
return where, nil
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf(
|
||||||
|
"cannot read the password this provisioner was given at %s: %w", path, err)
|
||||||
|
}
|
||||||
|
password := strings.TrimSpace(string(raw))
|
||||||
|
if password == "" {
|
||||||
|
// An empty file connects as nobody and is refused by the database, three layers from
|
||||||
|
// here, as an authentication problem with no cause anybody changed.
|
||||||
|
return "", fmt.Errorf("%s is empty, so this provisioner has no password", path)
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(where)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("MESH_PROVISION_POSTGRES is not a URL: %w", err)
|
||||||
|
}
|
||||||
|
user := parsed.User.Username()
|
||||||
|
if user == "" {
|
||||||
|
user = "postgres"
|
||||||
|
}
|
||||||
|
parsed.User = url.UserPassword(user, password)
|
||||||
|
return parsed.String(), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The password comes from a file, because that is how the mesh delivers one.
|
||||||
|
//
|
||||||
|
// A provisioner told to take a superuser password from an environment variable needs somebody to
|
||||||
|
// read the sealed file and pass it in — a person in the middle of the one path that exists so
|
||||||
|
// there is not one. It is also the difference between a credential in a file and one in a process
|
||||||
|
// listing: `docker inspect` prints environment.
|
||||||
|
func TestTheSuperuserPasswordComesFromTheFileTheMeshWrote(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "superuser")
|
||||||
|
if err := os.WriteFile(path, []byte("the-sealed-one\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres?sslmode=disable")
|
||||||
|
t.Setenv("MESH_PROVISION_PASSWORD_FILE", path)
|
||||||
|
|
||||||
|
where, err := connectionString()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(where, "the-sealed-one") {
|
||||||
|
t.Fatalf("the password the mesh wrote is not in the connection: %s", where)
|
||||||
|
}
|
||||||
|
if !strings.Contains(where, "127.0.0.1:5433") || !strings.Contains(where, "sslmode=disable") {
|
||||||
|
t.Fatalf("the rest of the connection was lost: %s", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An empty file connects as nobody and is refused by the database three layers away, as an
|
||||||
|
// authentication problem with no cause anybody changed.
|
||||||
|
func TestAnEmptyPasswordFileIsRefusedHere(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "superuser")
|
||||||
|
if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres@127.0.0.1:5433/postgres")
|
||||||
|
t.Setenv("MESH_PROVISION_PASSWORD_FILE", path)
|
||||||
|
if _, err := connectionString(); err == nil {
|
||||||
|
t.Fatal("a provisioner with no password reported one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a provisioner somebody runs by hand still works with the URL alone.
|
||||||
|
func TestAConnectionWithNoPasswordFileIsLeftAlone(t *testing.T) {
|
||||||
|
t.Setenv("MESH_PROVISION_POSTGRES", "postgres://postgres:typed@127.0.0.1:5433/postgres")
|
||||||
|
t.Setenv("MESH_PROVISION_PASSWORD_FILE", "")
|
||||||
|
where, err := connectionString()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if where != "postgres://postgres:typed@127.0.0.1:5433/postgres" {
|
||||||
|
t.Fatalf("the connection was rewritten when it should have been left alone: %s", where)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAProvisionerWithNoDatabaseSaysSo(t *testing.T) {
|
||||||
|
t.Setenv("MESH_PROVISION_POSTGRES", "")
|
||||||
|
t.Setenv("MESH_PROVISION_PASSWORD_FILE", "")
|
||||||
|
if _, err := connectionString(); err == nil {
|
||||||
|
t.Fatal("a provisioner that does not know which database it owns reported one")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PostgreSQL cuts an identifier at 63 bytes and says so only as a notice, so two consumers whose
|
||||||
|
// role names agree that far would quietly become one login — 022 again, at a length nobody tests.
|
||||||
|
func TestARoleNameTooLongToBeDistinctIsRefused(t *testing.T) {
|
||||||
|
if err := usableRole("mesh_anchor_gitea"); err != nil {
|
||||||
|
t.Fatalf("an ordinary name was refused: %v", err)
|
||||||
|
}
|
||||||
|
long := "mesh_" + strings.Repeat("n", 40) + "_" + strings.Repeat("m", 40)
|
||||||
|
err := usableRole(long)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("a role name PostgreSQL would shorten was accepted")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "share the login") {
|
||||||
|
t.Errorf("the refusal does not say what goes wrong: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The prefix this provisioner removes by is the prefix the mesh names by.
|
||||||
|
//
|
||||||
|
// **Two definitions on purpose.** A provisioner is a separate program and anyone may write one, so
|
||||||
|
// the prefix is part of the contract rather than a symbol to import — the same reason the grant
|
||||||
|
// file's shape is written down rather than shared. But a contract with two copies and no check is
|
||||||
|
// a contract until somebody edits one: if the mesh named `nox_` and this removed `mesh_`, every
|
||||||
|
// login it created would be permanent, and nothing would report anything at all.
|
||||||
|
func TestTheMarkAgreesWithWhatTheMeshNamesBy(t *testing.T) {
|
||||||
|
if mark != catalogue.IdentityPrefix {
|
||||||
|
t.Fatalf(
|
||||||
|
"this provisioner removes what begins with %q and the mesh names things %q, so it "+
|
||||||
|
"would never remove anything it made", mark, catalogue.IdentityPrefix)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// And a name the mesh would produce is one this provisioner accepts.
|
||||||
|
func TestWhatTheMeshNamesIsUsableAsARole(t *testing.T) {
|
||||||
|
if err := usableRole(catalogue.ConsumerIdentity("home-server", "keycloak")); err != nil {
|
||||||
|
t.Fatalf("the mesh named a consumer and this cannot make a role for it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
# The cache provisioner, as a module ships one.
|
||||||
|
#
|
||||||
|
# FROM scratch, like the postgres one and unlike the bucket one: it speaks the store's own wire
|
||||||
|
# protocol directly and needs no client in the image.
|
||||||
|
# The Go it builds with is the one the manifest pins (build.on GO_BASE), passed by the Makefile and the
|
||||||
|
# build machine alike; the default only serves a hand build, and matches go.mod.
|
||||||
|
ARG GO_BASE=golang:1.26-alpine
|
||||||
|
FROM ${GO_BASE} AS build
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' \
|
||||||
|
-o /mesh-provision-redis ./examples/redis-provisioner
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
COPY --from=build /mesh-provision-redis /mesh-provision-redis
|
||||||
|
# Watching by default, because that is what makes it a module: an ordinary long-running service
|
||||||
|
# the host supervises, rather than something invoked after every declaration.
|
||||||
|
ENTRYPOINT ["/mesh-provision-redis", "--watch"]
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user