Archived
Compare commits
1
Commits
main
..
64bc138692
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
64bc138692 |
+1
-8
@@ -2,11 +2,4 @@
|
||||
|
||||
# A built binary. The lab writes one here when pointed at the repository root by mistake;
|
||||
# built artifacts belong in build/, which is already ignored.
|
||||
/mesh-controller
|
||||
/postgres-provisioner
|
||||
/redis-provisioner
|
||||
# And where `go build` in cmd/mesh-controller leaves it (novox/hq issue 233).
|
||||
/cmd/mesh-controller/mesh-controller
|
||||
|
||||
# No repository commits its dependencies (issue 1305).
|
||||
vendor/
|
||||
/mesh-builder
|
||||
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# The Go it builds with, pinned here because genesis builds this file with no arguments (novox/hq
|
||||
# issue 223) — the Makefile passes the same digest. A tag older than go.mod asks for is how
|
||||
# `make image` broke once before (issue 146).
|
||||
ARG GO_BASE=golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
# The control plane's image — for genesis and the lab only. The mesh runs the controller as a Go
|
||||
# bundle the host starts as a process (module.json; novox/hq issue 213), and builds no image of it.
|
||||
# Genesis builds this file and raises it as the container the process replaces on the first push
|
||||
# (mesh-host internal/bootstrap, novox/hq issue 223).
|
||||
#
|
||||
# novox/hq ADR 0006: this image is pinned by digest in the bundle the host carries, fetched on a
|
||||
# machine where no mesh exists yet, and run before there is anything to check it against. So it
|
||||
# holds the program and nothing else — no shell, no package manager, no libc, nothing with a CVE
|
||||
# feed of its own. What a person has to audit before trusting a first node is one binary.
|
||||
#
|
||||
# There are no CA certificates in here on purpose. Nothing it does today makes an outbound TLS
|
||||
# connection to a public name: it reaches PostgreSQL on the machine it was raised on, and the
|
||||
# broker is verified against a fingerprint pinned in a token rather than against a public root
|
||||
# (novox/hq ADR 0004). Adding them "just in case" would put a trust store in the one image whose
|
||||
# whole argument is that it contains nothing to reason about.
|
||||
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
|
||||
# Dependencies first, so a change to the source does not refetch them.
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
ARG VERSION=development
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${VERSION}" \
|
||||
-o /mesh-controller ./cmd/mesh-controller
|
||||
|
||||
FROM scratch
|
||||
COPY --from=build /mesh-controller /mesh-controller
|
||||
|
||||
# Numeric because there is no /etc/passwd to look a name up in. Nothing here needs to be root:
|
||||
# it opens outbound connections and writes nothing to its own filesystem.
|
||||
USER 65534:65534
|
||||
|
||||
ENTRYPOINT ["/mesh-controller"]
|
||||
@@ -0,0 +1,141 @@
|
||||
# novox/hq ADR 0006 — the control plane, in Go.
|
||||
#
|
||||
# The image the bundle pins holds the program and nothing else, so the build is static and the
|
||||
# container is built FROM scratch. That is not a size optimisation: this image is fetched by
|
||||
# digest and run on a machine where no mesh exists to check anything, and everything in it is
|
||||
# something a person would have to audit.
|
||||
|
||||
VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo development)
|
||||
LDFLAGS := -s -w -X main.version=$(VERSION)
|
||||
|
||||
# Where `make check` raises PostgreSQL. A high port and a throwaway container: nothing here
|
||||
# touches a database anybody else is using. Override PG_PORT if this one is taken -- the first
|
||||
# port chosen was already serving something that had been up for six days.
|
||||
PG_PORT ?= 55532
|
||||
PG_CONTAINER ?= mesh-controller-check
|
||||
PG_IMAGE ?= postgres:17-alpine
|
||||
export MESH_TEST_POSTGRES ?= postgres://postgres:check@127.0.0.1:$(PG_PORT)/postgres?sslmode=disable
|
||||
|
||||
.PHONY: build image check test vet fmt postgres postgres-stop clean
|
||||
|
||||
build:
|
||||
CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o build/mesh-controller ./cmd/mesh-controller
|
||||
|
||||
# Tagged 'development' as well as by version, because the lab places images by name and a
|
||||
# scenario naming a version would have to be edited on every build. The version tag is what a
|
||||
# real bundle pins.
|
||||
IMAGE ?= mesh-controller:$(VERSION)
|
||||
DEV_TAG ?= mesh-controller:development
|
||||
|
||||
# The Go base the image is built on.
|
||||
#
|
||||
# **`make image` was broken and stayed broken**, because the Dockerfile's fallback base was a Go
|
||||
# older than go.mod asks for: every build died at `go mod download` with "go.mod requires go >=
|
||||
# 1.26.0", and the pipeline never saw it because the pipeline passes the declared base in. Anybody
|
||||
# building the image by hand hit it and had to find the digest themselves (novox/hq 04-ISSUES/146,
|
||||
# what it cost).
|
||||
#
|
||||
# **Pinned here since the manifest stopped building an image** (novox/hq issue 213): the mesh builds
|
||||
# the controller as a Go bundle with its own toolchain, and only `make image` — genesis and the lab —
|
||||
# still needs a Go base. The digest is the one the manifest declared until then.
|
||||
GO_BASE ?= golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c
|
||||
|
||||
image:
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) --build-arg VERSION=$(VERSION) -t $(IMAGE) -t $(DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The builder ships as an image too, because it is a module the mesh assigns rather than a program
|
||||
# somebody starts on a machine by hand.
|
||||
BUILDER_IMAGE ?= mesh-builder:$(VERSION)
|
||||
BUILDER_DEV_TAG ?= mesh-builder:development
|
||||
|
||||
builder-image:
|
||||
@test -n "$(GO_BASE)" || { echo "no GO_BASE; pass GO_BASE=<image>"; exit 1; }
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f cmd/mesh-builder/Dockerfile -t $(BUILDER_IMAGE) -t $(BUILDER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(BUILDER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The provisioner ships as an image too, because it is the thing that makes a sealed credential
|
||||
# true on a machine -- and the mesh cannot, having discarded the plaintext.
|
||||
PROVISIONER_IMAGE ?= mesh-provision-postgres:$(VERSION)
|
||||
PROVISIONER_DEV_TAG ?= mesh-provision-postgres:development
|
||||
|
||||
provisioner-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/postgres-provisioner/Dockerfile \
|
||||
-t $(PROVISIONER_IMAGE) -t $(PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The object store's provisioner, for the same reason: a bucket and a policy are not files, and
|
||||
# the mesh cannot make them -- it discarded the credential it would have to use.
|
||||
OBJECTSTORE_IMAGE ?= mesh-provision-objectstore:$(VERSION)
|
||||
OBJECTSTORE_DEV_TAG ?= mesh-provision-objectstore:development
|
||||
|
||||
objectstore-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/objectstore-provisioner/Dockerfile \
|
||||
-t $(OBJECTSTORE_IMAGE) -t $(OBJECTSTORE_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(OBJECTSTORE_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The cache's provisioner, for the same reason as the database's: an ACL user is not a file,
|
||||
# and the mesh cannot make one -- it discarded the credential it would have to use.
|
||||
REDIS_PROVISIONER_IMAGE ?= mesh-provision-redis:$(VERSION)
|
||||
REDIS_PROVISIONER_DEV_TAG ?= mesh-provision-redis:development
|
||||
|
||||
redis-provisioner-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/redis-provisioner/Dockerfile \
|
||||
-t $(REDIS_PROVISIONER_IMAGE) -t $(REDIS_PROVISIONER_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(REDIS_PROVISIONER_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The proxy that turns a route grant into traffic reaching a workload.
|
||||
PROXY_IMAGE ?= mesh-route-proxy:$(VERSION)
|
||||
PROXY_DEV_TAG ?= mesh-route-proxy:development
|
||||
|
||||
proxy-image:
|
||||
docker build --build-arg GO_BASE=$(GO_BASE) -f examples/route-proxy/Dockerfile -t $(PROXY_IMAGE) -t $(PROXY_DEV_TAG) .
|
||||
@echo
|
||||
@docker image inspect $(PROXY_IMAGE) --format 'built {{.RepoTags}} {{.Size}} bytes'
|
||||
|
||||
# The whole gate. Raises a database, runs everything against it, and takes it down again --
|
||||
# including when the tests fail, which is why the teardown is not conditional.
|
||||
#
|
||||
# **One package at a time (-p 1), and it is not about speed.** The live tests reach one bus, and on
|
||||
# it they assert, read and remove the mesh's own objects -- streams and consumers with fixed names,
|
||||
# because those names are the mesh's and a test cannot choose others. Two packages doing that at once
|
||||
# is one deleting a consumer the other is reading through, and the failure lands in whichever test
|
||||
# was reading, as "no response from stream". That reads as a bug in the code under test.
|
||||
check: fmt vet postgres
|
||||
@go test -p 1 ./... ; status=$$? ; $(MAKE) postgres-stop ; exit $$status
|
||||
|
||||
# Without a database the live tests skip rather than fail, so this is the honest subset and not
|
||||
# the gate. Serialised for the same reason check is: a bus may be configured even when a store is not.
|
||||
test:
|
||||
go test -p 1 ./...
|
||||
|
||||
vet:
|
||||
go vet ./...
|
||||
|
||||
fmt:
|
||||
@unformatted=$$(gofmt -l . 2>/dev/null) ; \
|
||||
if [ -n "$$unformatted" ] ; then echo "not gofmt'd:" ; echo "$$unformatted" ; exit 1 ; fi
|
||||
|
||||
postgres:
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
@docker run -d --name $(PG_CONTAINER) -e POSTGRES_PASSWORD=check \
|
||||
-p 127.0.0.1:$(PG_PORT):5432 $(PG_IMAGE) >/dev/null
|
||||
@printf 'waiting for postgres'
|
||||
@for i in $$(seq 1 60) ; do \
|
||||
if docker exec $(PG_CONTAINER) pg_isready -U postgres >/dev/null 2>&1 ; then \
|
||||
echo ' — ready' ; exit 0 ; fi ; \
|
||||
printf '.' ; sleep 1 ; \
|
||||
done ; \
|
||||
echo ' — never came up' ; docker logs $(PG_CONTAINER) | tail -20 ; exit 1
|
||||
|
||||
postgres-stop:
|
||||
@docker rm -f $(PG_CONTAINER) >/dev/null 2>&1 || true
|
||||
|
||||
clean:
|
||||
rm -rf build/
|
||||
@@ -1,7 +1,218 @@
|
||||
# mesh-controller
|
||||
|
||||
The controller ended under issue 1126. Its jobs moved to the parts that ADR 0590 and ADR 0593 name: the indexer, the
|
||||
assigner, the provisioner, the grants part and the node-engine. Its last module, route-proxy, moved to mesh-networking.
|
||||
**Tier 2 of Novox Mesh — the control plane.** Everything that needs to know about more than one
|
||||
node.
|
||||
|
||||
This repository holds no code and is archived once the delivery of the group that emptied it (release sdk1) is done.
|
||||
Its history stays here for reading.
|
||||
That is the whole test, and it draws the line the host cannot: the host applies and does not
|
||||
decide, *because deciding needs knowledge one machine does not have*. Which nodes should run the
|
||||
store, which peers belong in an overlay, whether a node has been unreachable for a week — nobody
|
||||
on a single machine can answer any of them.
|
||||
|
||||
The reasoning lives in [novox/hq](https://git.novox.be/novox/hq); this repository carries no
|
||||
argument that is not settled there.
|
||||
|
||||
## What it is not
|
||||
|
||||
- **Not the thing that changes machines.** It decides; the host applies. It never reaches into a
|
||||
node except through the host, over the link, in a bounded vocabulary.
|
||||
- **Not a database.** There is no mesh database. Each context owns its own store and nothing
|
||||
outside a context touches it — including nodes, which hold no credential to any of them.
|
||||
- **Not privileged.** It has no more access to a machine than a declaration can express.
|
||||
|
||||
## What exists today
|
||||
|
||||
**One context of seven, and one of the things it will do.**
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| `inventory` | node records and enrolment tokens — **built, as far as identity** |
|
||||
| `identity` | the control plane's signing key, and the keys nodes are known by — **built** |
|
||||
| `config`, `connectivity`, `provisioning`, `delivery`, `observability` | not built |
|
||||
| the interface every surface speaks to | not built; its shape is not decided |
|
||||
|
||||
```
|
||||
mesh-controller migrate bring each context's schema up to date
|
||||
mesh-controller node add <name> create a node record
|
||||
mesh-controller node list the nodes this mesh knows about
|
||||
mesh-controller token issue --node <name> a one-time right to join, for an existing record
|
||||
mesh-controller token issue --new <name> create the record and issue for it
|
||||
mesh-controller identity show this control plane's signing key
|
||||
mesh-controller broker show where the broker is, and what to expect there
|
||||
mesh-controller version what this binary is
|
||||
```
|
||||
|
||||
`migrate` is **step 3 of the foundation bootstrap** — the step the first node cannot get past, run
|
||||
against a database raised moments earlier from the bundle the host carries.
|
||||
|
||||
### Tokens, and what they are missing
|
||||
|
||||
A token is **a one-time right to join, issued for a node record** — which is where re-enrolment is
|
||||
decided, since what an identity binds to is settled when the token is made rather than when it is
|
||||
presented.
|
||||
|
||||
What is built: the secret is 256 bits from the system's random source, shown once, and **stored
|
||||
only as a hash**, so a copy of this database is not a set of working credentials. It is usable
|
||||
exactly once and only before it expires, and both are read from the row rather than from a status
|
||||
something would have had to write. Issuing again for the same node invalidates the outstanding
|
||||
one — two live tokens are two machines able to join as the same node.
|
||||
|
||||
Redemption is a single statement that both finds a live token and spends it, so eight concurrent
|
||||
attempts on one secret produce exactly one winner. There is a test that runs them.
|
||||
|
||||
**A token now carries three of its four parts**, and is one line of base64 a person can copy. The
|
||||
signing key is real: an Ed25519 key this control plane generates once and keeps, whose public half
|
||||
travels in every token. A node believes a declaration because it carries a signature that key made
|
||||
— and pinning only the broker would not do, because it would make the control plane's authority
|
||||
transitive, so a compromised broker could forge declarations, and since the host applies whatever
|
||||
the link delivers that is the whole machine.
|
||||
|
||||
**All four parts are built.** Given `MESH_BROKER_ADDRESS` and `MESH_BROKER_CERTIFICATE`, a token
|
||||
carries everything ADR 0004 requires. Without them it carries two, and `token issue` prints it
|
||||
**and names what is missing** rather than producing something that looks complete and cannot be
|
||||
used.
|
||||
|
||||
**The fingerprint is derived from the certificate, never configured.** A configured one can drift
|
||||
from the certificate it describes, and a drifted pin is worse than none: every node issued a token
|
||||
during the drift refuses to connect, and the failure looks like an attack. It is computed over the
|
||||
DER bytes — what a client actually sees on the wire — so the same certificate re-wrapped with
|
||||
different line endings still produces the same pin.
|
||||
|
||||
### Two contexts, and the rule between them is real
|
||||
|
||||
`identity` is the second context and it exists partly to test a claim this repository had made and
|
||||
never checked: that a context reaches only its own store. It holds `MESH_STORE_IDENTITY`;
|
||||
`inventory` holds `MESH_STORE_INVENTORY`; there is no setting that reaches both and no way to ask
|
||||
for one. Run `migrate` with only one and it stops, naming the grant it does not have.
|
||||
|
||||
A token needs a node record from one and a signing key from the other. Neither reads the other's
|
||||
store — the process holding both grants asks each for its part.
|
||||
|
||||
### How a node is known
|
||||
|
||||
**The node generates a keypair; the mesh records the public half.** The same principle as SSH, and
|
||||
the same rule `08-connectivity` already applies to the overlay keys — which is where this was
|
||||
settled all along, though it took being asked directly to notice.
|
||||
|
||||
Only the public half is ever stored, and that is the property worth having: **a copy of this
|
||||
database grants nothing.** It is a list of who to believe, not a set of credentials, which is what
|
||||
makes *compromise of a node is compromise of that node* literally true.
|
||||
|
||||
Exactly one key is live per node. Re-enrolment revokes the one it replaced, in the same
|
||||
transaction — two live identities for one node record is the stolen-laptop case, with the replaced
|
||||
machine still believed. The database enforces it as well as the code, and there is a test running
|
||||
six concurrent enrolments that fails when the constraint is removed.
|
||||
|
||||
**Not the machine's SSH host key**, though that was the obvious economy. Host keys are regenerated
|
||||
by reinstalls and image clones, which would silently un-enrol a node; their lifecycle belongs to
|
||||
sshd rather than the mesh; and a partial host has no SSH daemon at all.
|
||||
|
||||
## Reaching a store
|
||||
|
||||
**A context is granted only what it exclusively owns.** No shared writes, no read-only role on
|
||||
another context's store, and no connection string that reaches more than one.
|
||||
|
||||
That is a rule about credentials, so it is built as one. There is no mesh-wide connection setting
|
||||
and no way to ask for one:
|
||||
|
||||
```
|
||||
MESH_STORE_INVENTORY=postgres://…/inventory
|
||||
```
|
||||
|
||||
A process granted `inventory` holds that variable and no other. Reaching another context's store
|
||||
is not a matter of restraint — it has no address for it and no credential to present. And it is
|
||||
how the rule is *checked*: what a context can reach is visible in the declaration that runs it,
|
||||
as the list of variables it was given.
|
||||
|
||||
Each context's database is named after the context. There is deliberately no database named for
|
||||
the mesh as a whole.
|
||||
|
||||
## Migrations
|
||||
|
||||
Numbered, embedded in the binary, applied in order, each in a transaction with the row recording
|
||||
it. The applying is four lines; the rest is refusals, and the refusals are the point:
|
||||
|
||||
| it stops when | because |
|
||||
|---|---|
|
||||
| a migration that ran has since been edited | the database holds the old version, the repository holds the new one, and nothing holds the difference |
|
||||
| a migration is numbered below one that already ran | usually two branches taking the same next number — applying it now runs the schema in an order nobody tested |
|
||||
| two migrations share a number | order is the entire guarantee, and two files with one number have none |
|
||||
| a file in the directory is not a valid migration name | a misnamed migration would otherwise never run and nothing would say so |
|
||||
| a migration is empty | it records that something happened and changes nothing, which cannot be told from a mistake |
|
||||
|
||||
All of them stop rather than warn. At the moment any of them is true, nobody knows what the
|
||||
database contains, and there is no correct guess about a schema.
|
||||
|
||||
Running it again does nothing. Two copies running at once take an advisory lock, so a restart
|
||||
during a slow migration does not become two runners racing.
|
||||
|
||||
## Building
|
||||
|
||||
```
|
||||
make build the binary
|
||||
make image the container image
|
||||
make check gofmt, vet, and every test against a real PostgreSQL
|
||||
```
|
||||
|
||||
`make check` raises a throwaway PostgreSQL in a container and takes it down afterwards, including
|
||||
when the tests fail. Without one the tests that need a database **skip and say so** rather than
|
||||
passing quietly — `make test` is the honest subset, not the gate.
|
||||
|
||||
Tests run against a real database rather than a fake because what is being tested *is* the
|
||||
database's behaviour: that DDL is transactional, that an advisory lock serialises, that a
|
||||
checksum mismatch is caught against a record PostgreSQL actually kept. A fake would assert that
|
||||
the fake behaves as expected.
|
||||
|
||||
Every test here has been confirmed to fail when the behaviour it defends is removed. Two did not,
|
||||
when first written, and both are now commented with what they were missing.
|
||||
|
||||
## Checks that cross into the host's repository
|
||||
|
||||
Two things are agreed between this repository and `novox/mesh-host`, and each is a separate struct
|
||||
on each side. A field renamed on one of them fails **silently** — the crossing succeeds and
|
||||
something is simply absent — so both are checked by handing one side's real output to the other's
|
||||
real parser. Neither runs by default; each skips with a reason, because a repository that fails
|
||||
without its neighbour checked out is a repository nobody can build.
|
||||
|
||||
**What this mesh sends, read by the host that receives it:**
|
||||
|
||||
```
|
||||
mesh-controller: ./build/mesh-controller plan <node> --json > /tmp/d.json
|
||||
mesh-host: MESH_EMITTED=/tmp/d.json go test ./internal/declaration/ -v
|
||||
```
|
||||
|
||||
**What a node says when it joins, read by this mesh:**
|
||||
|
||||
```
|
||||
mesh-host: MESH_ENROL_OUT=/tmp/enrol.json go test ./internal/link/
|
||||
mesh-controller: MESH_ENROL=/tmp/enrol.json make check
|
||||
```
|
||||
|
||||
The second does more than compare shapes: it seals something to the key that arrived and opens it
|
||||
with the private half the host kept. A key that is present, correctly named and simply *wrong*
|
||||
passes every check that only looks at the message.
|
||||
|
||||
## The image
|
||||
|
||||
**The mesh no longer runs the controller from it** (novox/hq issue 213). The module declares a Go
|
||||
bundle, `controller`, which the host on the controller's machine unpacks and runs as the process
|
||||
`mesh-controller` under the account of the same name (ADR 0188 §1, §3). The image stays for what
|
||||
still runs a container of the controller: genesis, which raises the first controller from it and
|
||||
installs the module from its manifest (mesh-host `internal/bootstrap`), and the lab. Neither is the
|
||||
mesh's own build any more — `make image` builds it.
|
||||
|
||||
`FROM scratch`, holding one statically linked binary and nothing else — no shell, no package
|
||||
manager, no libc, no CA certificates.
|
||||
|
||||
Not a size optimisation. The bundle a host carries pins this image by digest, and it is fetched
|
||||
and run on a machine where no mesh exists to check anything and a person is expected to have read
|
||||
the bundle and believed it. Everything in the image is something that person would have to audit.
|
||||
|
||||
## Where the reasoning lives
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| what the control plane is | novox/hq ADR 0006 |
|
||||
| what it takes to run one, and why Go | novox/hq ADR 0006 |
|
||||
| a context owns its store, exclusively | novox/hq ADR 0008 |
|
||||
| schema changes are numbered migrations | novox/hq ADR 0013 |
|
||||
| a test defends a decision | novox/hq ADR 0017 |
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
ARG ALPINE_BASE=alpine:3
|
||||
ARG GO_BASE=golang:1.25-alpine
|
||||
# The builder, as a module ships one.
|
||||
#
|
||||
# Not FROM scratch, unlike the control plane: this one runs git and a container client, so it
|
||||
# needs a filesystem with them on it. That is the honest cost of a machine whose job is to build —
|
||||
# and it is why building is a MODULE on a machine that has a runtime rather than something the
|
||||
# control plane does (novox/hq ADR 0005).
|
||||
FROM ${GO_BASE} AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags '-s -w' -o /mesh-builder ./cmd/mesh-builder
|
||||
|
||||
FROM ${ALPINE_BASE}
|
||||
# git to clone what it is asked to build, and the docker client to build and push it. The daemon
|
||||
# is the machine's, reached through its socket — a build machine shares the runtime it was given
|
||||
# rather than running one inside itself.
|
||||
RUN apk add --no-cache git docker-cli
|
||||
COPY --from=build /mesh-builder /usr/local/bin/mesh-builder
|
||||
ENTRYPOINT ["/usr/local/bin/mesh-builder"]
|
||||
@@ -0,0 +1,386 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// What a holder of the build seat answers for, on its own machine (novox/hq ADR 0219).
|
||||
//
|
||||
// **The queue is the controller's; the build running here is this machine's.** The controller can
|
||||
// see and change what waits in the seat's queue, but an ask a holder already took is a process tree
|
||||
// on this machine and containers in this machine's runtime, and only this machine can end them. So
|
||||
// the holder serves four verbs on the seat's subjects for this machine: what it is building, kill
|
||||
// it, pause, resume.
|
||||
//
|
||||
// **One build at a time** (ADR 0190), which is also what builder.Said assumes — a package global
|
||||
// set per build — so "the build running here" is one or none, and kill names it by id so a call
|
||||
// that arrives as one build ends and the next begins cannot end the wrong one.
|
||||
|
||||
// holder is this machine's state as a holder of the build seat.
|
||||
type holder struct {
|
||||
on, seat string
|
||||
// workspace is where the paused flag is kept, so a holder restarted while paused stays paused
|
||||
// rather than silently taking work again.
|
||||
workspace string
|
||||
// say publishes this machine's state: whether it takes work (link.HolderState).
|
||||
say func(link.HolderState) error
|
||||
// remove runs what a kill needs outside the build: the containers left behind.
|
||||
remove builder.Runner
|
||||
|
||||
mu sync.Mutex
|
||||
paused bool
|
||||
running *running
|
||||
}
|
||||
|
||||
// running is the build this machine is doing.
|
||||
type running struct {
|
||||
request link.BuildRequest
|
||||
step string
|
||||
started time.Time
|
||||
cancel context.CancelFunc
|
||||
killed bool
|
||||
// returned is the build's own work having ended, before a kill or not; outcome is what was then
|
||||
// announced, and announced whether it went out.
|
||||
returned bool
|
||||
outcome string
|
||||
announced bool
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// pausedFile is where the flag lives in the workspace.
|
||||
func pausedFile(workspace string) string { return filepath.Join(workspace, ".mesh-builder-paused") }
|
||||
|
||||
// newHolder reads the paused flag the workspace keeps.
|
||||
func newHolder(on, seat, workspace string, say func(link.HolderState) error) *holder {
|
||||
h := &holder{on: on, seat: seat, workspace: workspace, say: say, remove: plainRun}
|
||||
if _, err := os.Stat(pausedFile(workspace)); err == nil {
|
||||
h.paused = true
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Paused is asked by the taking loop before every fetch.
|
||||
func (h *holder) Paused() bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.paused
|
||||
}
|
||||
|
||||
// setPaused records the flag in the workspace first and then in memory, so what this holder says
|
||||
// it is and what it would be after a restart never differ.
|
||||
func (h *holder) setPaused(paused bool) error {
|
||||
path := pausedFile(h.workspace)
|
||||
if paused {
|
||||
if err := os.MkdirAll(h.workspace, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
|
||||
return fmt.Errorf("cannot keep the paused flag in %s: %w", path, err)
|
||||
}
|
||||
} else if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("cannot remove the paused flag %s: %w", path, err)
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.paused = paused
|
||||
h.mu.Unlock()
|
||||
h.announce()
|
||||
return nil
|
||||
}
|
||||
|
||||
// announce says whether this machine takes work. Never fatal: the flag is kept either way, and the
|
||||
// controller reading an older state is a plan read as late rather than a build lost.
|
||||
func (h *holder) announce() {
|
||||
if h.say == nil {
|
||||
return
|
||||
}
|
||||
if err := h.say(link.HolderState{On: h.on, Paused: h.Paused(), At: time.Now().UTC().Format(time.RFC3339Nano)}); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say whether this machine takes builds: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// stateWhilePaused says this machine's state at start, and again every while it stays paused, so
|
||||
// a pause outlives the events stream's retention.
|
||||
func (h *holder) stateWhilePaused(ctx context.Context, every time.Duration) {
|
||||
h.announce()
|
||||
tick := time.NewTicker(every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
if h.Paused() {
|
||||
h.announce()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// begin records the build this machine took, with the cancel that ends it.
|
||||
func (h *holder) begin(request link.BuildRequest, cancel context.CancelFunc) *running {
|
||||
r := &running{request: request, started: time.Now(), cancel: cancel, done: make(chan struct{})}
|
||||
h.mu.Lock()
|
||||
h.running = r
|
||||
h.mu.Unlock()
|
||||
return r
|
||||
}
|
||||
|
||||
// end clears it, and lets a kill waiting on it know it is over.
|
||||
func (h *holder) end(r *running) {
|
||||
h.mu.Lock()
|
||||
if h.running == r {
|
||||
h.running = nil
|
||||
}
|
||||
h.mu.Unlock()
|
||||
close(r.done)
|
||||
}
|
||||
|
||||
// stepped records the step a build is at, for `current`.
|
||||
func (h *holder) stepped(r *running, step string) {
|
||||
h.mu.Lock()
|
||||
r.step = step
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// currentBuild is what `current` answers.
|
||||
type currentBuild struct {
|
||||
On string `json:"on"`
|
||||
Paused bool `json:"paused"`
|
||||
Running *struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
} `json:"running,omitempty"`
|
||||
Said string `json:"said"`
|
||||
}
|
||||
|
||||
func (h *holder) current() currentBuild {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
out := currentBuild{On: h.on, Paused: h.paused}
|
||||
taking := "taking builds"
|
||||
if h.paused {
|
||||
taking = "paused, taking no new build"
|
||||
}
|
||||
if h.running == nil {
|
||||
out.Said = fmt.Sprintf("%s is building nothing; %s", h.on, taking)
|
||||
return out
|
||||
}
|
||||
r := h.running
|
||||
elapsed := time.Since(r.started).Round(time.Second)
|
||||
out.Running = &struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Step string `json:"step,omitempty"`
|
||||
Started string `json:"started"`
|
||||
Elapsed string `json:"elapsed"`
|
||||
}{r.request.ID, r.request.Repository, r.request.Path, r.request.Ref, r.step,
|
||||
r.started.UTC().Format(time.RFC3339), elapsed.String()}
|
||||
out.Said = fmt.Sprintf("%s is building %s (%s) at %s for %s; %s",
|
||||
h.on, r.request.Repository, r.request.ID, orNothing(r.step), elapsed, taking)
|
||||
return out
|
||||
}
|
||||
|
||||
// The bounds a kill keeps: each pass removing containers, and the wait for the build to end between
|
||||
// them. The worst case — 15s, 20s, 15s — is inside what the controller waits for the answer
|
||||
// (killAnswer in the controller's queue.go, 75s).
|
||||
var (
|
||||
killRemoves = 15 * time.Second
|
||||
killWaits = 20 * time.Second
|
||||
)
|
||||
|
||||
// kill ends the build with this id, if it is the one running here and still working: its context
|
||||
// cancelled — which kills each command's process group — and the containers it started removed by
|
||||
// their label, once at once and again after the build has ended, so one created while it was being
|
||||
// killed is not left. The build's own goroutine announces it failed, killed by hand, and settles the
|
||||
// ask so it is not redelivered; the answer says whether that happened.
|
||||
func (h *holder) kill(id string) (string, error) {
|
||||
h.mu.Lock()
|
||||
r := h.running
|
||||
if r == nil || r.request.ID != id {
|
||||
doing := "nothing"
|
||||
if r != nil {
|
||||
doing = r.request.ID
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s is not building %s; it is building %s. `queue` says where an ask is", h.on, id, doing)
|
||||
}
|
||||
if r.returned {
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("%s on %s has already ended on its own and is saying how; `builds` shows it", id, h.on)
|
||||
}
|
||||
r.killed = true
|
||||
cancel, done := r.cancel, r.done
|
||||
h.mu.Unlock()
|
||||
|
||||
cancel()
|
||||
removed, removeErr := h.removeContainers(id)
|
||||
ended := false
|
||||
select {
|
||||
case <-done:
|
||||
ended = true
|
||||
case <-time.After(killWaits):
|
||||
}
|
||||
again, againErr := h.removeContainers(id)
|
||||
removed += again
|
||||
if removeErr == nil {
|
||||
removeErr = againErr
|
||||
}
|
||||
containers := fmt.Sprintf("%d container(s) it started removed", removed)
|
||||
if removeErr != nil {
|
||||
containers = "its containers could not all be listed or removed: " + removeErr.Error()
|
||||
}
|
||||
if !ended {
|
||||
return fmt.Sprintf("killed %s on %s: %s; the build has not finished ending yet — `builds` says when "+
|
||||
"its outcome is in", id, h.on, containers), nil
|
||||
}
|
||||
h.mu.Lock()
|
||||
outcome, announced := r.outcome, r.announced
|
||||
h.mu.Unlock()
|
||||
if !announced {
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended and %s, and its outcome could not be "+
|
||||
"announced — the ask is not settled and will be handed out again", id, r.request.Repository, h.on,
|
||||
containers), nil
|
||||
}
|
||||
return fmt.Sprintf("killed %s (%s) on %s: its commands ended, %s, and its outcome announced as failed, %s — "+
|
||||
"settled, so it is not handed to another machine", id, r.request.Repository, h.on, containers, outcome), nil
|
||||
}
|
||||
|
||||
// removeContainers is one pass of removing what the build left, bounded.
|
||||
func (h *holder) removeContainers(id string) (int, error) {
|
||||
cleanup, stop := context.WithTimeout(context.Background(), killRemoves)
|
||||
defer stop()
|
||||
return builder.RemoveContainersOf(cleanup, h.remove, id)
|
||||
}
|
||||
|
||||
// returned records that the build's work ended, and says whether a kill came first — only then is
|
||||
// the build killed; an error it ended with on its own is its own outcome.
|
||||
func (h *holder) returned(r *running) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
r.returned = true
|
||||
return r.killed
|
||||
}
|
||||
|
||||
// said records the outcome announced, and whether it went out, for a kill to answer with.
|
||||
func (h *holder) said(r *running, outcome string, announced bool) {
|
||||
h.mu.Lock()
|
||||
r.outcome, r.announced = outcome, announced
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// handlers are the seat's verbs, as this machine answers them.
|
||||
func (h *holder) handlers() map[string]link.ToolHandler {
|
||||
return map[string]link.ToolHandler{
|
||||
"current": func(context.Context, json.RawMessage) (any, error) { return h.current(), nil },
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &args); err != nil || strings.TrimSpace(args.ID) == "" {
|
||||
return nil, errors.New("kill needs the build's id")
|
||||
}
|
||||
said, err := h.kill(strings.TrimSpace(args.ID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": said}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(true); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
said := h.on + " is paused: it takes no new build until resumed"
|
||||
if c := h.current(); c.Running != nil {
|
||||
said += "; " + c.Running.ID + " runs on and finishes"
|
||||
}
|
||||
return map[string]any{"said": said, "paused": true}, nil
|
||||
},
|
||||
"resume": func(context.Context, json.RawMessage) (any, error) {
|
||||
if err := h.setPaused(false); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"said": h.on + " takes builds again", "paused": false}, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "its start"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// plainRun runs a command outside any build: no line reaches a build's log, because the build whose
|
||||
// log it would be is the one being ended.
|
||||
func plainRun(ctx context.Context, dir, name string, args ...string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, name, args...)
|
||||
cmd.Dir = dir
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return string(out), fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// announcement is what this holder answers discovery with (novox/hq ADR 0195, ADR 0197): this
|
||||
// machine's verbs of the seat, in the shape every tool runtime announces a seat's verb — kind seat,
|
||||
// the module answering, the seat, scope node, the machine, its description and argument schema — so
|
||||
// the console finds `<node>/node-build-agent.kill` by searching, as it finds any seat's verb.
|
||||
//
|
||||
// One service per machine, named for the seat and identified by the machine, so the answer is this
|
||||
// machine's four verbs and nothing more. The verbs are the compiled seat row's: a build machine has no
|
||||
// store, and what it serves is what this binary was built to serve.
|
||||
func announcement(seat, module, node string) micro.Info {
|
||||
s, _ := catalogue.SeatNamed(seat)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, v := range s.Serves {
|
||||
schema, _ := json.Marshal(v.Input)
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: seat + "__" + v.Name,
|
||||
Subject: link.NodeSeatToolSubject(seat, v.Name, node),
|
||||
// The queue group the verbs are served in, as every runtime announces its own.
|
||||
QueueGroup: "seat." + seat,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": module, "tool": v.Name, "seat": seat, "scope": "node",
|
||||
"node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{Name: seat, ID: node, Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": seat, "scope": "node", "node": node, "module": module}},
|
||||
Description: "what the build running on " + node + " is, and ending, pausing and resuming it (novox/hq ADR 0219)",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
|
||||
// moduleOf is the module a credential was issued for: its user is `<node>.<module>`.
|
||||
func moduleOf(user string) string {
|
||||
if _, module, ok := strings.Cut(user, "."); ok && module != "" {
|
||||
return module
|
||||
}
|
||||
return "build-agent"
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A holder paused stays paused across its restart: the flag is kept in its workspace (novox/hq ADR
|
||||
// 0219), and what it says about itself follows.
|
||||
func TestAPausedHolderStaysPausedAcrossARestart(t *testing.T) {
|
||||
workspace := t.TempDir()
|
||||
var said []link.HolderState
|
||||
h := newHolder("ace", link.TheBuildMachine, workspace, func(s link.HolderState) error {
|
||||
said = append(said, s)
|
||||
return nil
|
||||
})
|
||||
if h.Paused() {
|
||||
t.Fatal("a new holder starts paused")
|
||||
}
|
||||
if _, err := h.handlers()["pause"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !h.Paused() || len(said) != 1 || !said[0].Paused || said[0].On != "ace" {
|
||||
t.Fatalf("paused: %v, said %+v", h.Paused(), said)
|
||||
}
|
||||
again := newHolder("ace", link.TheBuildMachine, workspace, nil)
|
||||
if !again.Paused() {
|
||||
t.Fatal("restarted, the holder forgot it was paused")
|
||||
}
|
||||
if _, err := again.handlers()["resume"](context.Background(), json.RawMessage(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if newHolder("ace", link.TheBuildMachine, workspace, nil).Paused() {
|
||||
t.Fatal("resumed, the holder came back paused")
|
||||
}
|
||||
}
|
||||
|
||||
// kill ends the build with that id — its context, which ends its commands — removes what it left by
|
||||
// label, and refuses an id it is not building.
|
||||
func TestKillEndsTheBuildRunningHereAndNoOther(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
var removed []string
|
||||
h.remove = func(_ context.Context, _ string, name string, args ...string) (string, error) {
|
||||
removed = append(removed, name+" "+strings.Join(args, " "))
|
||||
if args[0] == "ps" {
|
||||
return "c1\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
kill := h.handlers()["kill"]
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`)); err == nil {
|
||||
t.Fatal("killed a build while none ran")
|
||||
}
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1", Repository: "novox/a"}, cancel)
|
||||
h.stepped(r, "image")
|
||||
if c := h.current(); c.Running == nil || c.Running.ID != "build-1" || c.Running.Step != "image" {
|
||||
t.Fatalf("current says %+v", c)
|
||||
}
|
||||
if _, err := kill(context.Background(), json.RawMessage(`{"id":"build-2"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "build-1") {
|
||||
t.Fatalf("killed another id: %v", err)
|
||||
}
|
||||
// The build's own goroutine: it ends when its context does, as a build's commands do, and
|
||||
// announces what came of it.
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, true)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
answer, err := kill(context.Background(), json.RawMessage(`{"id":"build-1"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if building.Err() == nil {
|
||||
t.Fatal("the build's context was not cancelled")
|
||||
}
|
||||
if !r.killed {
|
||||
t.Error("the build is not marked killed, so it would be announced as an ordinary failure")
|
||||
}
|
||||
said := answer.(map[string]any)["said"].(string)
|
||||
if !strings.Contains(said, "2 container(s)") || !strings.Contains(said, "announced as failed") || !strings.Contains(said, link.KilledByHand) {
|
||||
t.Errorf("kill said %q", said)
|
||||
}
|
||||
// Removed at the kill and again once the build had ended: a container made in between is caught.
|
||||
if len(removed) != 4 || !strings.Contains(removed[0], "label=mesh.build=build-1") || !strings.Contains(removed[2], "label=mesh.build=build-1") {
|
||||
t.Errorf("removed %v", removed)
|
||||
}
|
||||
if c := h.current(); c.Running != nil {
|
||||
t.Errorf("after the kill current says %+v", c)
|
||||
}
|
||||
}
|
||||
|
||||
// A holder announces this machine's verbs of the seat as the console reads a seat's verb, and no more.
|
||||
func TestAHolderAnnouncesItsMachinesVerbsForTheConsole(t *testing.T) {
|
||||
info := announcement(link.TheBuildMachine, moduleOf("ace.build-agent"), "ace")
|
||||
if info.Name != "node-build-agent" || info.ID != "ace" || len(info.Endpoints) != 4 {
|
||||
t.Fatalf("announced %s/%s with %d endpoints", info.Name, info.ID, len(info.Endpoints))
|
||||
}
|
||||
kill := info.Endpoints[1]
|
||||
md := kill.Metadata
|
||||
if kill.Subject != "mesh.seat.node-build-agent.tool.kill.ace" || kill.QueueGroup != "seat.node-build-agent" || md["kind"] != "seat" || md["seat"] != "node-build-agent" ||
|
||||
md["scope"] != "node" || md["node"] != "ace" || md["tool"] != "kill" || md["module"] != "build-agent" ||
|
||||
!strings.Contains(md["description"], "killed by hand") || !strings.Contains(md["schema"], `"id"`) {
|
||||
t.Fatalf("kill is announced as %+v", kill)
|
||||
}
|
||||
body, err := json.Marshal(info)
|
||||
if err != nil || len(body) > 8*1024 {
|
||||
t.Fatalf("the answer is %d bytes (%v)", len(body), err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build that ended on its own as the kill arrived says what it did: the kill is refused, and its
|
||||
// own error is its outcome. One whose outcome could not be announced is not said to be settled.
|
||||
func TestAKillArrivingAfterTheBuildEndedIsRefusedAndAnUnannouncedKillSaysSo(t *testing.T) {
|
||||
h := newHolder("ace", link.TheBuildMachine, t.TempDir(), nil)
|
||||
h.remove = func(context.Context, string, string, ...string) (string, error) { return "", nil }
|
||||
_, cancel := context.WithCancel(context.Background())
|
||||
r := h.begin(link.BuildRequest{ID: "build-1"}, cancel)
|
||||
if killed := h.returned(r); killed {
|
||||
t.Fatal("a build nobody killed reads as killed")
|
||||
}
|
||||
if _, err := h.kill("build-1"); err == nil || !strings.Contains(err.Error(), "ended on its own") {
|
||||
t.Fatalf("killed a build that had ended: %v", err)
|
||||
}
|
||||
h.end(r)
|
||||
|
||||
building, cancel := context.WithCancel(context.Background())
|
||||
r = h.begin(link.BuildRequest{ID: "build-2"}, cancel)
|
||||
go func() {
|
||||
<-building.Done()
|
||||
if h.returned(r) {
|
||||
h.said(r, link.KilledByHand, false)
|
||||
}
|
||||
h.end(r)
|
||||
}()
|
||||
said, err := h.kill("build-2")
|
||||
if err != nil || strings.Contains(said, "announced as failed") || !strings.Contains(said, "could not be announced") {
|
||||
t.Fatalf("kill said %q (%v)", said, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,569 @@
|
||||
// mesh-builder — the thing a build machine runs.
|
||||
//
|
||||
// It takes work from the mesh, turns a repository into artifacts, publishes them, and says what
|
||||
// came out. It is **not** the control plane and it is **not** the host:
|
||||
//
|
||||
// - the control plane decides and never touches a machine. Building runs commands on one, and
|
||||
// what the control plane may send a machine is bounded by the declaration language
|
||||
// (novox/hq ADR 0005). "Run this build" is not in it, and widening the language so it could
|
||||
// be would make the control plane able to run anything anywhere.
|
||||
// - the host applies declarations and holds no opinion about what they contain. A host that
|
||||
// also built things would need a container runtime and git, on every machine, to do something
|
||||
// almost none of them will ever do.
|
||||
//
|
||||
// So it is a module: a program a machine runs because the mesh told it to, holding its own broker
|
||||
// credential and nothing else. Compromise of a build machine is compromise of a build machine.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// version is set at build time.
|
||||
var version = "development"
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-builder: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
const usage = `mesh-builder — builds modules for the mesh
|
||||
|
||||
It consumes build requests and answers with what it made. Nothing is listened on and nothing
|
||||
is dialled except the broker.
|
||||
|
||||
MESH_BROKER_FILE a file the mesh sealed to this machine holding the same
|
||||
MESH_REGISTRY host:port to publish artifacts to, when the mesh has not said
|
||||
MESH_BINDING a file the mesh wrote saying where the artifact store is
|
||||
MESH_PACKAGE_BINDING a file the mesh wrote saying where the package registry is
|
||||
MESH_NPM_TOKEN_FILE a file the mesh sealed holding the token for it
|
||||
MESH_NPM_REGISTRY a package registry URL, when the mesh has not said (a person, the bootstrap)
|
||||
MESH_NPM_TOKEN the token for it, likewise
|
||||
MESH_NPM_SCOPE the scope it answers for (default: @novox)
|
||||
MESH_WORKSPACE where to clone and build (default: a temporary directory)
|
||||
|
||||
It also builds one module and stops, which is how a mesh is raised — before there is a
|
||||
broker to take work from or a registry to publish into:
|
||||
|
||||
mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]
|
||||
|
||||
Without --registry the artifacts stay in this machine's container runtime, named by the
|
||||
digest of their own configuration. The result is printed as JSON.
|
||||
`
|
||||
|
||||
func run() error {
|
||||
if len(os.Args) > 1 {
|
||||
switch os.Args[1] {
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
case "build":
|
||||
return buildOnce(context.Background(), os.Args[2:])
|
||||
default:
|
||||
fmt.Print(usage)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
credential, err := brokerFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
registry, err := whereToPublish()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
workspace := os.Getenv("MESH_WORKSPACE")
|
||||
if workspace == "" {
|
||||
workspace = os.TempDir() + "/mesh-builder"
|
||||
}
|
||||
// **The mesh's name for this machine, not the container's.** A build is reported to the rest
|
||||
// of the mesh, and a report whose origin reads `104cb10e105b` names something no other module
|
||||
// can look up. The mesh already knows the answer and has a way to say it — `${machine:name}`
|
||||
// in the environment file this module is handed — so the hostname is only what is left when
|
||||
// nobody said.
|
||||
on := os.Getenv("MESH_NODE")
|
||||
if on == "" {
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
return fmt.Errorf("this build machine has no name: nothing said MESH_NODE and the host would not say either: %w", err)
|
||||
}
|
||||
on = hostname
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
js, seat, err := dialFor(credential)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// **This machine's holder: paused or not, and the build it is running** (novox/hq ADR 0219). The
|
||||
// paused flag is read from the workspace before anything is taken, so a holder restarted while
|
||||
// paused takes nothing.
|
||||
h := newHolder(on, seat, workspace, func(state link.HolderState) error {
|
||||
body, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Context().Publish(link.BuildPausedOf(seat, on), body)
|
||||
return err
|
||||
})
|
||||
if h.Paused() {
|
||||
fmt.Fprintf(os.Stderr, "paused (kept in %s): taking no build until resumed\n", pausedFile(workspace))
|
||||
}
|
||||
machine := link.MachineOverNATSWith(js, on, seat, link.MachineOptions{Paused: h.Paused})
|
||||
defer machine.Close()
|
||||
|
||||
// The seat's verbs, on this machine's subjects. Only the seat that declares them: the retired
|
||||
// one serves none, and a subscription its holder has no grant for would be refused for ever.
|
||||
if seat == link.TheBuildMachine {
|
||||
stopServing, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(seat, on, h.handlers(),
|
||||
log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopServing()
|
||||
// And says so, for the console to find (novox/hq ADR 0197).
|
||||
stopAnnouncing, err := link.OverNATS{Conn: js.Conn()}.Announce(
|
||||
announcement(seat, moduleOf(credential.User), on), log.New(os.Stderr, "", 0))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopAnnouncing()
|
||||
go h.stateWhilePaused(ctx, time.Hour)
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building for the mesh, publishing to %s\n", registry)
|
||||
publisher := builder.Registry{Address: registry, Run: builder.Command}
|
||||
|
||||
return machine.Take(ctx, func(ctx context.Context, work link.Build) {
|
||||
answer(ctx, publisher, on, workspace, work, h)
|
||||
})
|
||||
}
|
||||
|
||||
// dialFor opens this machine's link to whichever bus the mesh is on, and says which build seat it
|
||||
// holds.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5): a build
|
||||
// machine told about both would take work from one and answer on the other, and every log line would
|
||||
// say it was fine.
|
||||
func dialFor(credential Credential) (*broker.JetStream, string, error) {
|
||||
// **The credential names the bus, and there is one** (novox/hq ADR 0131, design 28 task 5.5).
|
||||
// A credential for the mesh's bus carries user, password and fingerprint beside the address,
|
||||
// and that is enough to dial it, pinned.
|
||||
if !credential.onTheNewBus() {
|
||||
return nil, "", fmt.Errorf("the credential at hand names %q, which is not the mesh's bus", credential.URL)
|
||||
}
|
||||
js, err := broker.DialPinned(credential.natsURL(), credential.Fingerprint)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// **The seat this machine serves is the one its credential claims** (novox/hq ADR 0190, the
|
||||
// handover): the mesh issues a build machine's credential naming the seat its module claims,
|
||||
// and one binary serves the old role as `builder` and the new as `build-agent` from that alone.
|
||||
seat := link.BuildSeatClaimed(credential.seatsClaimed())
|
||||
fmt.Fprintf(os.Stderr, "taking build work as a holder of %s\n", seat)
|
||||
return js, seat, nil
|
||||
}
|
||||
|
||||
// answer does one build and says what happened, whichever way it went.
|
||||
func answer(ctx context.Context, publisher builder.Publisher, on, workspace string, work link.Build, h *holder) {
|
||||
request := work.Request()
|
||||
|
||||
// **Its own context, so it can be killed alone** (novox/hq ADR 0219): cancelled by `kill`, it ends
|
||||
// this build's commands and nothing else; the machine's own context ending — a SIGTERM — still
|
||||
// reaches it through the parent, and that keeps today's meaning below.
|
||||
building, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var mine *running
|
||||
if h != nil {
|
||||
mine = h.begin(request, cancel)
|
||||
defer h.end(mine)
|
||||
}
|
||||
|
||||
// **First thing, and to stdout.** A build request that arrives and produces no visible line until
|
||||
// it either finishes or fails is indistinguishable from one that never arrived — which cost a long
|
||||
// diagnosis against a running mesh, chasing "the handler never fired" when the truth was only that
|
||||
// the handler said nothing until the end.
|
||||
fmt.Fprintf(os.Stderr, "a build request arrived for %s (%s)\n", request.Repository, request.ID)
|
||||
|
||||
// **Everything a build says goes two ways**: to stderr, as always, and onto the bus as the
|
||||
// role's own events under the build's id (novox/hq ADR 0157) — so whoever asked, and anybody
|
||||
// watching, reads the same lines this container's log holds, live, and after the fact from the
|
||||
// stream. Said first, before anything runs, so a build that hangs is one that visibly started.
|
||||
say := func(step, message string) {
|
||||
fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message)
|
||||
work.Say(step, message)
|
||||
if mine != nil && step != "run" && step != "output" {
|
||||
h.stepped(mine, step)
|
||||
}
|
||||
}
|
||||
builder.Said = say
|
||||
defer func() { builder.Said = nil }()
|
||||
if err := work.Began(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot say a build started: %v\n", err)
|
||||
}
|
||||
|
||||
result := link.BuildResult{
|
||||
ID: request.ID, Repository: request.Repository, Path: request.Path,
|
||||
Ref: request.Ref, On: on, Source: request.Source, DryRun: request.DryRun,
|
||||
}
|
||||
what := "building " + request.Repository
|
||||
if request.Path != "" {
|
||||
what += " at " + request.Path
|
||||
}
|
||||
if request.Ref != "" {
|
||||
what += " on " + request.Ref
|
||||
}
|
||||
say("build", what)
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
var built builder.Result
|
||||
if err == nil {
|
||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||
// build that could not have resolved its dependencies is refused in front of the reason, not
|
||||
// after a clone that then fails at npm ci.
|
||||
// Every container it starts is labelled with its id, so a kill finds what outlived the
|
||||
// docker client (ADR 0219).
|
||||
built, err = builder.Build(building, builder.Labelled(builder.Command, request.ID), publisher,
|
||||
request.Repository, request.Path, request.Ref, workspace, request.Held, npmrc,
|
||||
forgeFrom(), say, request.Seats)
|
||||
}
|
||||
// Only a build the kill ended: the kill came before its work did. One that finished — built, or
|
||||
// failed on its own — in the moment the kill arrived says what it did, and the kill is refused.
|
||||
killed := false
|
||||
if mine != nil {
|
||||
killed = h.returned(mine) && err != nil
|
||||
}
|
||||
if killed {
|
||||
// **Killed by hand is the outcome, whatever the build was doing** (novox/hq ADR 0219): the
|
||||
// error it ended with is the kill's consequence, not a fault of the source.
|
||||
result.Failed = link.KilledByHand
|
||||
say("failed", link.KilledByHand)
|
||||
} else if err != nil {
|
||||
// A failure is a result. A build that fails and says nothing is indistinguishable from a
|
||||
// builder that is not running, and those want completely different responses.
|
||||
result.Failed = err.Error()
|
||||
say("failed", err.Error())
|
||||
} else {
|
||||
manifest, marshalErr := json.Marshal(built.Manifest)
|
||||
if marshalErr != nil {
|
||||
result.Failed = marshalErr.Error()
|
||||
} else {
|
||||
result.Commit = built.Commit
|
||||
result.Manifest = manifest
|
||||
for _, made := range built.Built {
|
||||
result.Made = append(result.Made, link.MadeArtifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
}
|
||||
result.Against = built.Against
|
||||
for _, r := range built.Read {
|
||||
result.Read = append(result.Read, link.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
say("built", built.Manifest.Module+" from "+short(built.Commit))
|
||||
}
|
||||
}
|
||||
|
||||
// A killed build is announced on a context of its own: the build's was the one cancelled, and the
|
||||
// outcome must go out and the ask be settled — acknowledged, never redelivered to another machine
|
||||
// to be built again. A machine being stopped is the other case and keeps its meaning: the
|
||||
// parent's context is gone, nothing is announced or settled, and the ask is redelivered.
|
||||
announcing := ctx
|
||||
if killed {
|
||||
fresh, stop := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer stop()
|
||||
announcing = fresh
|
||||
}
|
||||
announceErr := work.Announce(announcing, result)
|
||||
if mine != nil {
|
||||
h.said(mine, result.Failed, announceErr == nil)
|
||||
}
|
||||
if err := announceErr; err != nil {
|
||||
// Said, not fatal: the build happened. A build reported as failed because announcing it
|
||||
// failed is a lie about work that was done — and the request stays unsettled below only if
|
||||
// nothing was said at all, so another machine can try.
|
||||
fmt.Fprintf(os.Stderr, "cannot say what came of a build: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Settled only once the outcome is away, so a machine that dies before answering leaves the work
|
||||
// for another rather than losing it.
|
||||
if err := work.Done(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the outcome is away and the request could not be settled: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// packagesFrom is where a build resolves the mesh's own published packages — the SDK above all
|
||||
// (novox/hq ADR 0076, issue 053).
|
||||
//
|
||||
// Preferably from the mesh: a package-registry binding names the endpoint the way the artifact
|
||||
// store's binding does, and a sealed token file the credential the way the broker's does. The
|
||||
// environment variables remain for a builder run by a person, and for the bootstrap, where there is
|
||||
// no registry yet — there the result is disabled and a build that needs no mesh-published dependency
|
||||
// builds anyway.
|
||||
func packagesFrom() (builder.Npmrc, error) {
|
||||
scope := strings.TrimSpace(os.Getenv("MESH_NPM_SCOPE"))
|
||||
if scope == "" {
|
||||
scope = "@novox"
|
||||
}
|
||||
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_NPM_REGISTRY"))
|
||||
var username string
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read what the mesh said about the package registry: %w", err)
|
||||
}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s is not a binding: %w", path, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
return builder.Npmrc{}, fmt.Errorf(
|
||||
"%s says the package registry is on %q and gives no address for it", path, told.From)
|
||||
}
|
||||
// Composed from what the provider serves, so nothing here knows gitea's URL shape from
|
||||
// another registry's: it states its port, the path its registry answers on, and the scheme.
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about which port the package registry answers on", path)
|
||||
}
|
||||
npmPath, ok := told.Serves["npm-path"]
|
||||
if !ok {
|
||||
return builder.Npmrc{}, fmt.Errorf("%s says nothing about the path the package registry answers on", path)
|
||||
}
|
||||
registry = fmt.Sprintf("%s://%s:%v%v", scheme, told.At, port, npmPath)
|
||||
username = told.As
|
||||
}
|
||||
|
||||
// The credential the mesh sealed to this machine. The mesh authenticates the ordinary way — a
|
||||
// generated password the provider only applies (novox/hq ADR 0048) — so with a username this is
|
||||
// a password (basic auth); without one it is a bearer token a provider minted.
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.Npmrc{}, fmt.Errorf("cannot read this builder's package-registry credential: %w", err)
|
||||
}
|
||||
secret = strings.TrimSpace(string(raw))
|
||||
}
|
||||
if u := strings.TrimSpace(os.Getenv("MESH_NPM_USER")); u != "" {
|
||||
username = u
|
||||
}
|
||||
|
||||
if registry == "" && secret == "" {
|
||||
return builder.Npmrc{}, nil
|
||||
}
|
||||
if username != "" {
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Username: username, Password: secret}, nil
|
||||
}
|
||||
return builder.Npmrc{Scope: scope, Registry: registry, Token: secret}, nil
|
||||
}
|
||||
|
||||
// forgeFrom is the git credential this builder may offer a clone, composed from the same binding
|
||||
// and sealed secret its package-registry half already reads: the forge that answers npm is the
|
||||
// forge that hosts the repositories, and its provisioner applies one password to one user for
|
||||
// both. Anything missing means no credential, and every clone stays anonymous — which is all a
|
||||
// mesh of public repositories ever needs.
|
||||
//
|
||||
// The URL names the binding's own address — the machine the mesh says the forge is on — so a
|
||||
// private repository is registered and built by that address, and a clone of anything else is
|
||||
// never shown this credential (git's credential store matches the whole origin).
|
||||
func forgeFrom() builder.GitCredential {
|
||||
path := strings.TrimSpace(os.Getenv("MESH_PACKAGE_BINDING"))
|
||||
if path == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
var told struct {
|
||||
At string `json:"at"`
|
||||
As string `json:"as"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil || told.At == "" || told.As == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
secret := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN"))
|
||||
if file := strings.TrimSpace(os.Getenv("MESH_NPM_TOKEN_FILE")); file != "" {
|
||||
if raw, err := os.ReadFile(file); err == nil {
|
||||
secret = strings.TrimSpace(string(raw))
|
||||
}
|
||||
}
|
||||
if secret == "" {
|
||||
return builder.GitCredential{}
|
||||
}
|
||||
scheme := "https"
|
||||
if s, ok := told.Serves["scheme"]; ok {
|
||||
scheme = fmt.Sprintf("%v", s)
|
||||
}
|
||||
host := told.At
|
||||
if port, ok := told.Serves["port"]; ok {
|
||||
host = fmt.Sprintf("%s:%v", told.At, port)
|
||||
}
|
||||
made := url.URL{Scheme: scheme, User: url.UserPassword(told.As, secret), Host: host}
|
||||
return builder.GitCredential{URL: made.String()}
|
||||
}
|
||||
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
|
||||
// whereToPublish is the artifact store this builder uses.
|
||||
//
|
||||
// **Preferably from the mesh.** A builder that is a module requires an artifact store, and the
|
||||
// mesh writes it a file saying which machine answers that and on what port — the same binding any
|
||||
// consumer of any provision gets. Reading it means the address is not a setting somebody keeps in
|
||||
// step by hand, and moving the store is an ordinary reassignment rather than an edit on every
|
||||
// build machine.
|
||||
//
|
||||
// The environment variable remains for a builder run by a person, which is how this started and
|
||||
// how it is still run while being developed.
|
||||
func whereToPublish() (string, error) {
|
||||
binding := strings.TrimSpace(os.Getenv("MESH_BINDING"))
|
||||
if binding == "" {
|
||||
registry := strings.TrimSpace(os.Getenv("MESH_REGISTRY"))
|
||||
if registry == "" {
|
||||
return "", fmt.Errorf("neither MESH_BINDING nor MESH_REGISTRY: a built artifact " +
|
||||
"nobody can fetch is not built")
|
||||
}
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(binding)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("cannot read what the mesh said about the artifact store: %w", err)
|
||||
}
|
||||
var told struct {
|
||||
From string `json:"from"`
|
||||
At string `json:"at"`
|
||||
Serves map[string]any `json:"serves"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &told); err != nil {
|
||||
return "", fmt.Errorf("%s is not a binding: %w", binding, err)
|
||||
}
|
||||
if told.At == "" {
|
||||
// The provider is not on the private network, so there is no name to reach it by. Said
|
||||
// rather than falling back to the machine's own name, which would publish to a store on
|
||||
// the wrong machine and be found out much later.
|
||||
return "", fmt.Errorf(
|
||||
"%s says the artifact store is on %q and gives no address for it", binding, told.From)
|
||||
}
|
||||
port, ok := told.Serves["port"]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s says nothing about which port the artifact store answers on",
|
||||
binding)
|
||||
}
|
||||
return fmt.Sprintf("%s:%v", told.At, port), nil
|
||||
}
|
||||
|
||||
// brokerFrom is where this builder connects, and with what.
|
||||
//
|
||||
// **Preferably from a file the mesh sealed to this machine.** A builder that is a module is given
|
||||
// its credential the way every other module is given one: generated or accepted centrally, sealed
|
||||
// to the machine, written by the host. Putting it in an environment variable instead would mean
|
||||
// the one copy that matters passing through a terminal and a process listing.
|
||||
//
|
||||
// The variable remains for a builder run by a person.
|
||||
func brokerFrom() (Credential, error) {
|
||||
if path := strings.TrimSpace(os.Getenv("MESH_BROKER_FILE")); path != "" {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Credential{}, fmt.Errorf("cannot read this builder's credential: %w", err)
|
||||
}
|
||||
said := strings.TrimSpace(string(raw))
|
||||
if said == "" {
|
||||
// An empty credential file is a machine that will connect as nobody and be refused,
|
||||
// with the reason three layers away.
|
||||
return Credential{}, fmt.Errorf("%s is empty, so this builder has no credential", path)
|
||||
}
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(said), &held); err == nil && held.URL != "" {
|
||||
return held, nil
|
||||
}
|
||||
// A file holding only a URL, which is what a person writing one by hand produces. The
|
||||
// broker is then verified against whatever this machine already trusts.
|
||||
return Credential{URL: said}, nil
|
||||
}
|
||||
return Credential{}, fmt.Errorf(
|
||||
"no MESH_BROKER_FILE: a build machine with no credential for the bus has nothing to build")
|
||||
}
|
||||
|
||||
// Credential is what a build machine is given so it can reach the broker.
|
||||
//
|
||||
// Two things, because reaching a broker over TLS needs both: who to connect as, and what to check
|
||||
// the certificate against. A mesh's broker presents a certificate of the mesh's own, which is in
|
||||
// no public trust store, so a URL alone can only connect to a broker somebody else vouches for.
|
||||
//
|
||||
// **The same shape a node gets, for the same reason** (novox/hq ADR 0004): the fingerprint travels
|
||||
// out of band — here, sealed with the credential — and the endpoint is verified once at connect.
|
||||
type Credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
// User and Password ride beside the address on the bus being built (design 25): a credential
|
||||
// embedded in a URL leaks into every log line that prints a connection, so the mesh seals them
|
||||
// as two fields and this machine joins them once, here, to dial.
|
||||
User string `json:"user,omitempty"`
|
||||
Password string `json:"password,omitempty"`
|
||||
// Claims are the seats the module this credential was issued for claims, as the mesh writes
|
||||
// them beside the credential (novox/hq ADR 0159). The first is the build role this machine
|
||||
// serves; a credential naming none is from before claims travelled in it.
|
||||
Claims []struct {
|
||||
Seat string `json:"seat"`
|
||||
} `json:"claims,omitempty"`
|
||||
}
|
||||
|
||||
// seatsClaimed is the seats the credential names, in order.
|
||||
func (c Credential) seatsClaimed() []string {
|
||||
out := make([]string, 0, len(c.Claims))
|
||||
for _, claim := range c.Claims {
|
||||
out = append(out, claim.Seat)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onTheNewBus is whether a credential is for the bus being built: its address says so, and the
|
||||
// mesh only ever seals such a credential with the user and password beside it.
|
||||
func (c Credential) onTheNewBus() bool { return strings.HasPrefix(strings.TrimSpace(c.URL), "nats://") }
|
||||
|
||||
// natsURL is the address with this machine's credential in it, for the one dial that needs it.
|
||||
func (c Credential) natsURL() string {
|
||||
rest := strings.TrimPrefix(strings.TrimSpace(c.URL), "nats://")
|
||||
if c.User == "" {
|
||||
return "nats://" + rest
|
||||
}
|
||||
return "nats://" + c.User + ":" + c.Password + "@" + rest
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
)
|
||||
|
||||
// buildOnce is the builder doing one build and stopping, with no broker and no mesh.
|
||||
//
|
||||
// **This is how a mesh is raised** (novox/hq ADR 0073). The installer carries this program and runs
|
||||
// it once, before anything else exists, to produce the control plane from the same repository and
|
||||
// path that every later rebuild of it will use. What raises the mesh is therefore the same thing
|
||||
// that maintains it — not a second mechanism that has to be kept in step with the first and is
|
||||
// exercised once per new mesh, which is how often enough to rot.
|
||||
//
|
||||
// It takes no work from a queue and answers nobody: there is no broker yet, and the only thing
|
||||
// waiting for the answer is the installer that started it. So the result goes to standard output as
|
||||
// JSON, which is what the installer reads.
|
||||
//
|
||||
// With no --registry it publishes nowhere and the image stays in this machine's container runtime,
|
||||
// named by the digest of its own configuration (see builder.Local). That is the genesis case. With
|
||||
// one, it publishes as it always does — the same command is how an operator builds a module by hand
|
||||
// on a mesh that already exists.
|
||||
func buildOnce(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||
path := set.String("path", "", "the module's directory inside the repository (default: its root)")
|
||||
ref := set.String("ref", "", "the commit to build; a branch is a moving target somebody else controls")
|
||||
registry := set.String("registry", "",
|
||||
"host:port to publish to. Without it the artifacts stay in this machine's container runtime, which is the genesis case")
|
||||
workspace := set.String("workspace", "", "where to clone and build (default: a temporary directory)")
|
||||
on := multiple(set, "on",
|
||||
"a base this build stands on, as <module>/<artifact>=<reference>. Repeatable")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("mesh-builder build <repository> [--path P] [--ref COMMIT] [--registry HOST:PORT]")
|
||||
}
|
||||
repository := positionals[0]
|
||||
|
||||
where := *workspace
|
||||
if where == "" {
|
||||
where = os.TempDir() + "/mesh-builder-once"
|
||||
}
|
||||
|
||||
// What this build may stand on, said on the command line because there is no mesh to ask.
|
||||
//
|
||||
// **This is the genesis case and it is meant to be awkward.** On a running mesh the control
|
||||
// plane answers this, because only it knows what this mesh holds. Here nothing has been built
|
||||
// yet, so whoever runs this says what to use — and for the control plane, which is the one
|
||||
// module raised before anything else exists, the answer is ordinarily nothing at all.
|
||||
bases := map[string]string{}
|
||||
for _, pair := range *on {
|
||||
key, reference, found := strings.Cut(pair, "=")
|
||||
if !found || key == "" || reference == "" {
|
||||
return fmt.Errorf(
|
||||
"--on takes <module>/<artifact>=<reference>, and %q is not that", pair)
|
||||
}
|
||||
bases[key] = reference
|
||||
}
|
||||
|
||||
// Local unless told otherwise, because the moment this exists for has nowhere to publish. A
|
||||
// default pointing at a registry would mean genesis failing at a push to something that is not
|
||||
// there yet, one step away from the thing that could explain it.
|
||||
var publisher builder.Publisher = builder.Local{Run: builder.Command}
|
||||
if *registry != "" {
|
||||
publisher = builder.Registry{Address: *registry, Run: builder.Command}
|
||||
}
|
||||
|
||||
fmt.Fprintf(os.Stderr, "building %s", repository)
|
||||
if *path != "" {
|
||||
fmt.Fprintf(os.Stderr, " at %s", *path)
|
||||
}
|
||||
if *ref != "" {
|
||||
fmt.Fprintf(os.Stderr, " at %s", *ref)
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
|
||||
npmrc, err := packagesFrom()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
built, buildErr := builder.Build(ctx, builder.Command, publisher, repository, *path, *ref, where, bases, npmrc,
|
||||
forgeFrom(),
|
||||
func(step, message string) { fmt.Fprintf(os.Stderr, " [%s] %s\n", step, message) })
|
||||
if buildErr != nil {
|
||||
return buildErr
|
||||
}
|
||||
|
||||
// To standard output, and everything else to standard error, so the caller can read this
|
||||
// without having to separate it from progress.
|
||||
out := onceResult{
|
||||
Module: built.Manifest.Module,
|
||||
Commit: built.Commit,
|
||||
Repository: repository,
|
||||
Path: *path,
|
||||
Ref: *ref,
|
||||
Manifest: built.Manifest,
|
||||
Against: built.Against,
|
||||
}
|
||||
for _, r := range built.Read {
|
||||
out.Read = append(out.Read, readRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
for _, made := range built.Built {
|
||||
out.Made = append(out.Made, madeArtifact{Name: made.Name, Kind: made.Kind, Reference: made.Reference})
|
||||
}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
fmt.Fprintf(os.Stderr, " built %s from %s\n", built.Manifest.Module, short(built.Commit))
|
||||
return nil
|
||||
}
|
||||
|
||||
// onceResult is what one build reports to whoever started it.
|
||||
//
|
||||
// The same fields the mesh records for a build, so a reader comparing a genesis build against an
|
||||
// ordinary one is comparing the same thing said the same way.
|
||||
type onceResult struct {
|
||||
Module string `json:"module"`
|
||||
Commit string `json:"commit"`
|
||||
Repository string `json:"repository"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
Manifest any `json:"manifest"`
|
||||
Made []madeArtifact `json:"made"`
|
||||
Against []string `json:"against,omitempty"`
|
||||
Read []readRepository `json:"read,omitempty"`
|
||||
}
|
||||
|
||||
// readRepository is a repository this build read source from besides the module's own.
|
||||
type readRepository struct {
|
||||
Repository string `json:"repository"`
|
||||
Ref string `json:"ref,omitempty"`
|
||||
}
|
||||
|
||||
type madeArtifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Reference string `json:"reference"`
|
||||
}
|
||||
|
||||
// parseAround lets flags appear on either side of the repository, because a person writing this by
|
||||
// hand will put them wherever reads best and the standard parser stops at the first thing that is
|
||||
// not a flag. The same helper the control plane's commands use, for the same reason.
|
||||
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
var positionals []string
|
||||
rest := args
|
||||
for {
|
||||
if err := set.Parse(rest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rest = set.Args()
|
||||
if len(rest) == 0 {
|
||||
return positionals, nil
|
||||
}
|
||||
positionals = append(positionals, rest[0])
|
||||
rest = rest[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// multiple is a flag that may be given more than once.
|
||||
type repeated []string
|
||||
|
||||
func (r *repeated) String() string { return strings.Join(*r, ", ") }
|
||||
func (r *repeated) Set(v string) error { *r = append(*r, v); return nil }
|
||||
|
||||
func multiple(set *flag.FlagSet, name, usage string) *[]string {
|
||||
var values repeated
|
||||
set.Var(&values, name, usage)
|
||||
return (*[]string)(&values)
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The seat a build machine serves comes from its credential (novox/hq ADR 0190 handover).
|
||||
func TestTheCredentialSaysWhichBuildRoleThisMachineServes(t *testing.T) {
|
||||
var held Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.builder","password":"x",
|
||||
"claims":[{"seat":"mesh-build-machine","scope":"mesh","serves":[]}]}`), &held); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(held.seatsClaimed()); got != "mesh-build-machine" {
|
||||
t.Errorf("the old builder's credential serves %q", got)
|
||||
}
|
||||
var bare Credential
|
||||
if err := json.Unmarshal([]byte(`{"url":"nats://bus:4222","user":"anchor.build-agent","password":"x"}`), &bare); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := link.BuildSeatClaimed(bare.seatsClaimed()); got != link.TheBuildMachine {
|
||||
t.Errorf("a credential without claims serves %q, want %s", got, link.TheBuildMachine)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **The genesis path writes its result to stdout and nothing else there.** The installer captures
|
||||
// stdout alone and parses it as JSON; one stray log line makes that fail with "invalid character"
|
||||
// — exactly what a builder fmt.Printf caused, breaking a build that had worked. This keeps
|
||||
// diagnostics on stderr so a future print cannot repeat it silently.
|
||||
func TestBuilderDiagnosticsStayOffStdout(t *testing.T) {
|
||||
allowed := map[string]bool{
|
||||
"string(body)": true, // once.go: the result JSON, which IS stdout
|
||||
"version)": true, // --version
|
||||
`"stopping")`: true, // the loop.s shutdown line
|
||||
"usage)": true, // --help text, for a human
|
||||
}
|
||||
for _, file := range []string{"once.go", "main.go"} {
|
||||
src, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, raw := range strings.Split(string(src), "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
if !strings.HasPrefix(line, "fmt.Printf(") &&
|
||||
!strings.HasPrefix(line, "fmt.Println(") &&
|
||||
!strings.HasPrefix(line, "fmt.Print(") {
|
||||
continue
|
||||
}
|
||||
ok := false
|
||||
for token := range allowed {
|
||||
if strings.Contains(line, token) {
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
t.Errorf("%s:%d writes a diagnostic to stdout, which the installer parses as JSON:\n %s",
|
||||
file, i+1, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/hex"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// Where a builder publishes.
|
||||
//
|
||||
// Preferably from the mesh: a builder that is a module requires an artifact store, and the mesh
|
||||
// writes it a binding saying which machine answers and on what port. Reading it means the address
|
||||
// is not a setting somebody keeps in step by hand.
|
||||
|
||||
func binding(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "artifact-store.json")
|
||||
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestTheMeshSaysWhereToPublish(t *testing.T) {
|
||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"provision":"artifact-store",
|
||||
"from":"anchor","at":"anchor.internal","serves":{"port":5000,"scheme":"http"}}`))
|
||||
where, err := whereToPublish()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if where != "anchor.internal:5000" {
|
||||
t.Fatalf("got %q", where)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABindingWithNoAddressIsRefused(t *testing.T) {
|
||||
// The provider is not on the private network, so there is no name to reach it by. Falling
|
||||
// back to anything would publish to a store on the wrong machine and be found out much later.
|
||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"anchor","serves":{"port":5000}}`))
|
||||
_, err := whereToPublish()
|
||||
if err == nil {
|
||||
t.Fatal("a binding with nowhere to reach was accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "anchor") {
|
||||
t.Fatalf("the failure does not name the machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABindingWithNoPortIsRefused(t *testing.T) {
|
||||
t.Setenv("MESH_BINDING", binding(t, `{"binding":1,"from":"a","at":"a.internal","serves":{}}`))
|
||||
if _, err := whereToPublish(); err == nil {
|
||||
t.Fatal("a binding saying nothing about a port was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheVariableStillWorksForABuilderRunByAPerson(t *testing.T) {
|
||||
// Which is how this started and how it is still run while being developed.
|
||||
t.Setenv("MESH_BINDING", "")
|
||||
t.Setenv("MESH_REGISTRY", "127.0.0.1:5000")
|
||||
where, err := whereToPublish()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if where != "127.0.0.1:5000" {
|
||||
t.Fatalf("got %q", where)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNeitherIsRefusedRatherThanGuessed(t *testing.T) {
|
||||
t.Setenv("MESH_BINDING", "")
|
||||
t.Setenv("MESH_REGISTRY", "")
|
||||
if _, err := whereToPublish(); err == nil {
|
||||
t.Fatal("a builder with nowhere to publish reported somewhere")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCredentialComesFromAFileTheMeshSealed(t *testing.T) {
|
||||
// A builder that is a module is given its credential the way every module is: sealed to the
|
||||
// machine and written by the host. An environment variable instead would put the one copy
|
||||
// that matters through a terminal and a process listing.
|
||||
path := filepath.Join(t.TempDir(), "broker")
|
||||
if err := os.WriteFile(path, []byte("amqps://a-builder:secret@broker.internal:5671/\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("MESH_BROKER_FILE", path)
|
||||
t.Setenv("MESH_BROKER_AMQP", "amqp://should-not-be-used@nowhere/")
|
||||
|
||||
got, err := brokerFrom()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
||||
t.Fatalf("got %q", got.URL)
|
||||
}
|
||||
}
|
||||
|
||||
// The credential the mesh seals carries what to check the broker's certificate against, because a
|
||||
// mesh's broker presents a certificate of the mesh's own and no public trust store has it. A URL
|
||||
// alone can only reach a broker somebody else vouches for.
|
||||
func TestTheSealedCredentialCarriesWhatVerifiesTheBroker(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "broker")
|
||||
if err := os.WriteFile(path, []byte(
|
||||
`{"url":"amqps://a-builder:secret@broker.internal:5671/","fingerprint":"abc123"}`),
|
||||
0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("MESH_BROKER_FILE", path)
|
||||
t.Setenv("MESH_BROKER_AMQP", "")
|
||||
|
||||
got, err := brokerFrom()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.URL != "amqps://a-builder:secret@broker.internal:5671/" {
|
||||
t.Fatalf("the url was lost: %q", got.URL)
|
||||
}
|
||||
if got.Fingerprint != "abc123" {
|
||||
t.Fatal("the builder was given nothing to check the broker against, so it can only " +
|
||||
"connect to a broker some public authority vouches for")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyCredentialFileIsRefused(t *testing.T) {
|
||||
// Otherwise the builder connects as nobody and is refused, with the reason three layers away.
|
||||
path := filepath.Join(t.TempDir(), "broker")
|
||||
if err := os.WriteFile(path, []byte("\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("MESH_BROKER_FILE", path)
|
||||
t.Setenv("MESH_BROKER_AMQP", "")
|
||||
if _, err := brokerFrom(); err == nil {
|
||||
t.Fatal("an empty credential was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuilderWithNoCredentialAtAllSaysSo(t *testing.T) {
|
||||
t.Setenv("MESH_BROKER_FILE", "")
|
||||
t.Setenv("MESH_BROKER_AMQP", "")
|
||||
if _, err := brokerFrom(); err == nil {
|
||||
t.Fatal("a builder with no broker reported one")
|
||||
}
|
||||
}
|
||||
|
||||
// The pin is compared in the spelling the mesh writes it.
|
||||
//
|
||||
// A bare digest against a written fingerprint never matches, and the failure is indistinguishable
|
||||
// from being pointed at the wrong broker — which is the one thing this check exists to report
|
||||
// truthfully. It cost a lab run.
|
||||
func TestThePinIsComparedInTheSpellingTheMeshWritesIt(t *testing.T) {
|
||||
certificate, key := aServerCertificate(t)
|
||||
der := certificate.Certificate[0]
|
||||
sum := sha256.Sum256(der)
|
||||
written := "sha256:" + hex.EncodeToString(sum[:])
|
||||
|
||||
listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{
|
||||
Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer listener.Close()
|
||||
go func() {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = conn.(*tls.Conn).Handshake()
|
||||
conn.Close()
|
||||
}
|
||||
}()
|
||||
_ = key
|
||||
|
||||
// The pin the mesh wrote must be accepted.
|
||||
if err := handshakeWith(listener.Addr().String(), written); err != nil {
|
||||
t.Fatalf("the broker this builder was told about was refused: %v", err)
|
||||
}
|
||||
// And a different one refused, or the check reports nothing.
|
||||
other := "sha256:" + strings.Repeat("ab", 32)
|
||||
if err := handshakeWith(listener.Addr().String(), other); err == nil {
|
||||
t.Fatal("a broker this builder was not told about was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// handshakeWith runs the pin check the builder dials with against an address.
|
||||
func handshakeWith(address, pin string) error {
|
||||
conn, err := tls.Dial("tcp", address, broker.PinnedToFingerprint(pin))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return conn.Close()
|
||||
}
|
||||
|
||||
func aServerCertificate(t *testing.T) (tls.Certificate, ed25519.PrivateKey) {
|
||||
t.Helper()
|
||||
public, private, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "a broker"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, public, private)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: private}, private
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The things the mesh can be asked to do, separated from how it was asked.
|
||||
//
|
||||
// **A surface is an adapter with no decisions in it** (novox/hq ADR 0035). The command line and
|
||||
// the command API call the same functions here, so an assignment refused at one is refused at the
|
||||
// other for the same reason and in the same words. The moment a surface can accept something
|
||||
// another would reject, the mesh has two answers to one question and people learn which to trust.
|
||||
//
|
||||
// Each returns what happened as text a person can read and a caller can pass on. Neither surface
|
||||
// composes its own explanation, because two explanations of one refusal drift.
|
||||
|
||||
// assign puts a module on a node, and says at once what in the mesh no longer works out.
|
||||
//
|
||||
// The assignment is kept even when the node does not resolve: it is what a person meant, and
|
||||
// assignment is not an ordering. A consumer assigned before its provider does not resolve for as
|
||||
// long as it takes to assign the provider, and refusing the first half of a pair would make the
|
||||
// order somebody types two commands in part of the mesh's rules.
|
||||
//
|
||||
// **What is checked is the mesh, not the one machine** — because that is what the assignment
|
||||
// changes. novox/hq 04-ISSUES/017 names the shape: an action can succeed into a state its own
|
||||
// verify rejects, and it happens when the action's test is not the test the verify uses. Here the
|
||||
// action tested one node and the verify is resolution over all of them, so an assignment could be
|
||||
// reported as fine while it took a provision away from every other machine — a module offering a
|
||||
// mesh-scoped provision stops offering it the moment its own node stops resolving, and every
|
||||
// consumer elsewhere is then told *nothing in this mesh provides it*, with a remedy that names a
|
||||
// module already assigned. That was found on a four-node raise and read as a version bump breaking
|
||||
// provider recognition; it was neither the version nor the provider.
|
||||
//
|
||||
// It costs a resolution per machine. Assignment is a person typing a command, and being told which
|
||||
// machines this just blocked is worth more than the milliseconds.
|
||||
func assign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("assign %s names no module", node)
|
||||
}
|
||||
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
|
||||
// be taken without ever having been previewed (novox/hq ADR 0100).
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
// **The one assignment refused for what the node lacks** (novox/hq ADR 0207). Everything else
|
||||
// an assignment leaves unresolved is kept, because assignment is not an ordering; a module whose
|
||||
// resources are applied through a seat nothing on the node holds is refused, because that order
|
||||
// — the service manager, the package manager and the runtime before anything that installs,
|
||||
// runs or contains — is the mesh's to keep. Several modules in one act are judged together, so
|
||||
// holders that depend on each other go on in one command.
|
||||
shelf, before, err := seatDependenciesOnAssign(ctx, open, node, modules)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// **Before the new assignment can unsettle a seat somebody holds only by being alone**
|
||||
// (novox/hq 04-ISSUES/170): what the mesh derived so far is written down, and then the
|
||||
// assignment resolves against a record rather than against a coincidence.
|
||||
settled, err := recordDerivedHolders(ctx, open)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var lines []string
|
||||
var added []string
|
||||
for _, module := range modules {
|
||||
fresh, err := open.inventory.Assign(ctx, node, module)
|
||||
if err != nil {
|
||||
return strings.Join(lines, "\n"), err
|
||||
}
|
||||
if !fresh {
|
||||
// Nothing changed, and saying "is assigned" would read as an action. One node runs one
|
||||
// of each — the module's name is the assignment's identity (novox/hq ADR 0115).
|
||||
lines = append(lines, fmt.Sprintf(
|
||||
"%s already runs %s — one node runs one of each (ADR 0115); nothing changed", node, module))
|
||||
continue
|
||||
}
|
||||
added = append(added, module)
|
||||
lines = append(lines, fmt.Sprintf("%s is assigned %s", node, module))
|
||||
}
|
||||
if len(added) == 0 {
|
||||
return strings.Join(lines, "\n"), nil
|
||||
}
|
||||
answer := strings.Join(lines, "\n")
|
||||
for _, line := range settled {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// What this act changed about this node's unmet seat dependencies, and nothing else (novox/hq
|
||||
// ADR 0207): a dependency of a module just assigned, or one this assignment met. The rest of the
|
||||
// node's list, and every other node's, is `status`'s.
|
||||
for _, line := range unheldChange(shelf, node, before, append(append([]string(nil), before...), added...)) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
// Its bus credential, in the same act (novox/hq issue 203): an assignment pushed before its
|
||||
// credential exists delivers a process that cannot authenticate and crash-loops until somebody
|
||||
// runs a second verb and a second push. Issued here when the module speaks on the bus and has
|
||||
// no credential yet; kept when it has one, so re-assigning rotates nothing.
|
||||
for _, module := range added {
|
||||
if line := issueOnAssign(ctx, open, node, module); line != "" {
|
||||
answer += "\n " + line
|
||||
}
|
||||
}
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
// Kept, and still refused. Both halves are the answer, and the rest of the mesh is still
|
||||
// worth reporting: this machine's refusal is rarely the only consequence.
|
||||
return answer + blockedElsewhere(ctx, open, node), err
|
||||
}
|
||||
// Kept, and cannot be hosted here. Said at once rather than discovered at push: a module whose
|
||||
// capability the machine lacks is on the wrong machine, and the assignment records what a person
|
||||
// meant while this line says it will not run until it moves. The rest of the node still pushes.
|
||||
isAdded := map[string]bool{}
|
||||
for _, m := range added {
|
||||
isAdded[m] = true
|
||||
}
|
||||
for _, u := range plan.Unhostable {
|
||||
if !isAdded[u.Module] {
|
||||
continue
|
||||
}
|
||||
for _, c := range u.Missing {
|
||||
answer += "\n but " + catalogue.WrongMachine(u.Module, c, node)
|
||||
}
|
||||
}
|
||||
return answer + fmt.Sprintf("\n run `push %s` to send it", node) +
|
||||
blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// seatDependenciesOnAssign is the refusal ADR 0207 makes at assignment, or nothing, with the
|
||||
// catalogue and the node's assignments it was judged against. Modules already assigned are not new
|
||||
// and are not judged again.
|
||||
func seatDependenciesOnAssign(ctx context.Context, open *stores, node string, modules []string) (
|
||||
map[string]catalogue.Manifest, []string, error) {
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
already := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
already[a] = true
|
||||
}
|
||||
var adding []string
|
||||
for _, m := range modules {
|
||||
if !already[m] {
|
||||
adding = append(adding, m)
|
||||
}
|
||||
}
|
||||
// The lines AssignRefusal says beside an assignment it lets through are said by unheldChange
|
||||
// with everything else this act changed, so they are not said twice.
|
||||
if _, err := catalogue.AssignRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Two modules declaring one package, path or unit is refused before anything is recorded
|
||||
// (novox/hq ADR 0210, 04-ISSUES/235): kept, the node would not resolve until one came off again.
|
||||
if err := catalogue.CollisionRefusal(shelf, node, assigned, adding); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return shelf, assigned, nil
|
||||
}
|
||||
|
||||
// unassign takes modules off a node. What they leave behind is the host's business: a directory
|
||||
// holding anything the mesh did not put there is kept (novox/hq ADR 0030).
|
||||
//
|
||||
// It reports the rest of the mesh for the same reason assign does, and more sharply: taking a
|
||||
// module off one machine is the ordinary way to stop providing something to another, and nothing
|
||||
// about the command's own output would ever have said so.
|
||||
//
|
||||
// **Refused when it takes away the last holder of a seat a module left on the node depends on**
|
||||
// (novox/hq ADR 0207) — the other side of refusing that module's assignment without one. Several
|
||||
// modules in one act are judged together, so a holder and its dependents come off in one command.
|
||||
func unassign(ctx context.Context, open *stores, node string, modules ...string) (string, error) {
|
||||
if len(modules) == 0 {
|
||||
return "", fmt.Errorf("unassign %s names no module", node)
|
||||
}
|
||||
ctx, release, err := holdNodes(ctx, open, []string{node})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// Every one checked before any is taken off, so a refusal leaves the node as it was.
|
||||
runs := map[string]bool{}
|
||||
for _, a := range assigned {
|
||||
runs[a] = true
|
||||
}
|
||||
for _, module := range modules {
|
||||
if !runs[module] {
|
||||
return "", fmt.Errorf("%s is not assigned to %s", module, node)
|
||||
}
|
||||
}
|
||||
if err := catalogue.UnassignRefusal(shelf, node, assigned, modules); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, module := range modules {
|
||||
if err := open.inventory.Unassign(ctx, node, module); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
answer := fmt.Sprintf("%s no longer runs %s — run `push %s` to make it so",
|
||||
node, strings.Join(modules, ", "), node)
|
||||
var left []string
|
||||
for _, a := range assigned {
|
||||
if !slices.Contains(modules, a) {
|
||||
left = append(left, a)
|
||||
}
|
||||
}
|
||||
for _, line := range unheldChange(shelf, node, assigned, left) {
|
||||
answer += "\n " + line
|
||||
}
|
||||
return answer + blockedElsewhere(ctx, open, node), nil
|
||||
}
|
||||
|
||||
// splitModules is a surface's one `module` field as the modules it names: several, comma-separated,
|
||||
// are one act (novox/hq ADR 0207), so the holders that depend on each other go on together from the
|
||||
// command API and the controller seat's verbs as they do from the command line.
|
||||
func splitModules(field string) []string {
|
||||
var out []string
|
||||
for _, m := range strings.Split(field, ",") {
|
||||
if m = strings.TrimSpace(m); m != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// blockedElsewhere is every OTHER machine that cannot be worked out as things now stand.
|
||||
//
|
||||
// **The state, not the cause.** Saying "this assignment broke laptop" would mean resolving the
|
||||
// whole mesh twice and would still be a guess about which of several changes did it; saying
|
||||
// "laptop cannot be worked out, and here is what it says" is true, is what somebody has to fix,
|
||||
// and cannot mislead. The machine that was just changed is left out because its own refusal is
|
||||
// already the answer beside this one.
|
||||
//
|
||||
// Nothing here can fail the act it reports on. A mesh that cannot be read is worth saying and is
|
||||
// not a reason to claim the assignment did not happen — it did.
|
||||
func blockedElsewhere(ctx context.Context, open *stores, except string) string {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return "\n\nThe rest of the mesh could not be checked: " + err.Error()
|
||||
}
|
||||
blocked := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if n.Name == except {
|
||||
continue
|
||||
}
|
||||
if _, _, err := planFor(ctx, open, n.Name); err != nil {
|
||||
blocked[n.Name] = err.Error()
|
||||
}
|
||||
}
|
||||
if len(blocked) == 0 {
|
||||
return ""
|
||||
}
|
||||
names := make([]string, 0, len(blocked))
|
||||
for name := range blocked {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
var out strings.Builder
|
||||
fmt.Fprintf(&out, "\n\nAND %d other machine(s) cannot be worked out as things stand, so "+
|
||||
"nothing will be sent to them:\n", len(names))
|
||||
for _, name := range names {
|
||||
fmt.Fprintf(&out, " %s\n", name)
|
||||
for _, line := range strings.Split(strings.TrimRight(blocked[name], "\n"), "\n") {
|
||||
fmt.Fprintf(&out, " %s\n", strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
out.WriteString("\nThis may or may not be what just changed — it is what is true now.")
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// issueOnAssign gives a newly assigned module its bus credential, the way `module issue` does, and
|
||||
// says what it did in one line. Nothing for a module that declares no broker secret; nothing for one
|
||||
// whose user is already minted (a credential is rotated on purpose, never by re-assigning); and when
|
||||
// the bus cannot be reached from here, the line names the verb and the push that would refuse the
|
||||
// module until it is run — never a silent placeholder (novox/hq issue 203).
|
||||
func issueOnAssign(ctx context.Context, open *stores, node, module string) string {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
m, known := shelf[module]
|
||||
if !known || mayIssue(m) != nil {
|
||||
return ""
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: module}.Username()
|
||||
if _, minted, err := inv.BusUserHash(ctx, user); err != nil || minted {
|
||||
return ""
|
||||
}
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err == nil {
|
||||
err = issueOnTheNewBus(ctx, inv, m, node, busAddress)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Sprintf("its bus credential is not issued (%v): `module issue %s --node %s` first — "+
|
||||
"`push %s` refuses to send %s until it is", err, module, node, node, module)
|
||||
}
|
||||
return fmt.Sprintf("its bus credential is issued and sealed to %s, and arrives with the push", node)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// What an assignment says about the machines it was not about.
|
||||
|
||||
// **An assignment that blocks another machine says so.**
|
||||
//
|
||||
// The shape found on a four-node raise: a module offering a mesh-scoped provision stops offering it
|
||||
// the moment its own node stops resolving, so an assignment to the provider's machine silently took
|
||||
// a provision away from every consumer elsewhere. Those consumers were then told *nothing in this
|
||||
// mesh provides it*, naming as the remedy a module that was already assigned — which read, on the
|
||||
// way back, as a version bump breaking provider recognition. It was neither the version nor the
|
||||
// provider: it was one machine's set of assignments, and nothing said so.
|
||||
//
|
||||
// novox/hq 04-ISSUES/017 names the general shape — an action succeeding into a state its own verify
|
||||
// rejects, because the action's test is not the test the verify uses. `assign` tested one node; the
|
||||
// verify is resolution over all of them.
|
||||
func TestAnAssignmentThatBlocksAnotherMachineSaysWhichAndWhy(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// A provider on the hub and a consumer on the other machine, both resolving.
|
||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||
Requires: []string{"acme-ca"}})
|
||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "route-proxy")
|
||||
if err != nil {
|
||||
t.Fatalf("a mesh that should resolve did not: %v\n%s", err, said)
|
||||
}
|
||||
if strings.Contains(said, "cannot be worked out") {
|
||||
t.Fatalf("a well mesh was reported as blocked:\n%s", said)
|
||||
}
|
||||
|
||||
// Now break the hub's own set, with nothing but the command a person has.
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
if _, err := assign(ctx, open, "anchor", "rival-one"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "anchor", "rival-two")
|
||||
if err == nil {
|
||||
t.Fatal("an assignment that makes its own node incoherent was not reported at all")
|
||||
}
|
||||
|
||||
// The node's own refusal is the error, as it always was. What is new is that the machines this
|
||||
// just took a provision away from are named in the same breath.
|
||||
if !strings.Contains(said, "laptop") {
|
||||
t.Fatalf("the machine this blocked is not named:\n%s\n\n%v", said, err)
|
||||
}
|
||||
if !strings.Contains(said, "acme-ca") {
|
||||
t.Fatalf("what laptop is now missing is not said:\n%s", said)
|
||||
}
|
||||
if !strings.Contains(said, "cannot be worked out as things stand") {
|
||||
t.Fatalf("the report does not say what state the mesh is in:\n%s", said)
|
||||
}
|
||||
// And the assignment is kept: it is what a person meant, and assignment is not an ordering.
|
||||
assigned, err := open.inventory.Assigned(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !contains(assigned, "rival-two") {
|
||||
t.Fatalf("the assignment was not kept: %v", assigned)
|
||||
}
|
||||
}
|
||||
|
||||
// A consumer assigned before its provider is refused for itself and kept, because assignment is not
|
||||
// an ordering — refusing the first half of a pair would make the order somebody types two commands
|
||||
// in part of the mesh's rules.
|
||||
func TestAConsumerAssignedBeforeItsProviderIsStillAssigned(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||
Requires: []string{"acme-ca"}})
|
||||
|
||||
said, err := assign(ctx, open, "laptop", "route-proxy")
|
||||
if err == nil {
|
||||
t.Fatalf("a consumer with nothing to consume resolved:\n%s", said)
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !contains(assigned, "route-proxy") {
|
||||
t.Fatalf("assignment became an ordering: %v", assigned)
|
||||
}
|
||||
}
|
||||
|
||||
// Unassigning is the ordinary way to stop providing something to another machine, and it reports
|
||||
// the same way for the same reason.
|
||||
func TestUnassigningAProviderNamesWhoIsNowBlocked(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "step-ca", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "acme-ca", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"acme-ca": {"path": "/acme/directory"}}})
|
||||
register(t, open, catalogue.Manifest{Module: "route-proxy", Version: "1",
|
||||
Requires: []string{"acme-ca"}})
|
||||
if _, err := assign(ctx, open, "anchor", "step-ca"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "route-proxy"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := unassign(ctx, open, "anchor", "step-ca")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "laptop") || !strings.Contains(said, "acme-ca") {
|
||||
t.Fatalf("taking the provider away said nothing about who was consuming it:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(all []string, one string) bool {
|
||||
for _, s := range all {
|
||||
if s == one {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// An address is read from the node's settings where it is used, never recorded with a port
|
||||
// (novox/hq 04-ISSUES/102). Three readers did not follow the setting; each is held to it here.
|
||||
|
||||
var aDigest = "sha256:" + strings.Repeat("e", 64)
|
||||
|
||||
// **A build is recorded by digest and path**, whatever address the builder pushed to — what it
|
||||
// made and what it stood on both; an image the module runs from elsewhere is left where it says.
|
||||
func TestABuildIsRecordedWithoutTheStoresAddress(t *testing.T) {
|
||||
manifest, _ := json.Marshal(map[string]any{
|
||||
"module": "gitea", "version": "1",
|
||||
"resources": []map[string]any{
|
||||
{"id": "server", "type": "container", "name": "mesh-gitea",
|
||||
"image": "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{"id": "config", "type": "archive", "path": "/etc/gitea", "digest": aDigest,
|
||||
"source": "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
{"id": "cache", "type": "container", "name": "mesh-gitea-cache",
|
||||
"image": "valkey/valkey@" + aDigest},
|
||||
},
|
||||
})
|
||||
kept := buildFrom(link.BuildResult{
|
||||
ID: "b1", Repository: "https://forge.example/gitea.git", Commit: "abc", On: "laptop",
|
||||
Manifest: manifest,
|
||||
Made: []link.MadeArtifact{
|
||||
{Name: "server", Kind: "image", Reference: "anchor.internal:5100/gitea/server@" + aDigest},
|
||||
{Name: "config", Kind: "archive", Reference: "http://anchor.internal:5100/v2/gitea/config/blobs/" + aDigest},
|
||||
},
|
||||
Against: []string{"anchor.internal:5100/mesh-tools/runtime@" + aDigest},
|
||||
})
|
||||
if kept.Module != "gitea" {
|
||||
t.Fatalf("the module was not read from the recorded manifest: %q", kept.Module)
|
||||
}
|
||||
if kept.Made[0].Reference != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the image is recorded as %q, address and all", kept.Made[0].Reference)
|
||||
}
|
||||
if kept.Made[1].Reference != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the archive is recorded as %q, address and all", kept.Made[1].Reference)
|
||||
}
|
||||
recorded, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recorded.Resources[0]["image"]; got != catalogue.ArtifactStoreScheme+"gitea/server@"+aDigest {
|
||||
t.Errorf("the recorded manifest's image is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[1]["source"]; got != catalogue.ArtifactStoreScheme+"gitea/config/blobs/"+aDigest {
|
||||
t.Errorf("the recorded manifest's archive is %v", got)
|
||||
}
|
||||
if got := recorded.Resources[2]["image"]; got != "valkey/valkey@"+aDigest {
|
||||
t.Errorf("an image the build did not make was rewritten: %v", got)
|
||||
}
|
||||
if strings.Contains(string(kept.Manifest), "anchor.internal:5100") {
|
||||
t.Errorf("the recorded manifest still carries the store's address:\n%s", kept.Manifest)
|
||||
}
|
||||
// What the build stood on is an edge to another module's artifact, and it is recorded the way
|
||||
// that artifact is: by path in the store, so the edge still names the same thing when the
|
||||
// store answers at another address.
|
||||
if kept.Against[0] != catalogue.ArtifactStoreScheme+"mesh-tools/runtime@"+aDigest {
|
||||
t.Errorf("what the build stood on was recorded by address: %v", kept.Against)
|
||||
}
|
||||
}
|
||||
|
||||
// aStore is a module offering the artifact store on 5000, published the long way as the
|
||||
// distribution module does, so a node may be given another number for it.
|
||||
func aStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "distribution", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: catalogue.ArtifactStoreProvision, Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{catalogue.ArtifactStoreProvision: {"port": float64(5000)}},
|
||||
Listens: []catalogue.Listening{{Port: 5000, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "store", "type": "container", "name": "mesh-registry",
|
||||
"ports": []any{"5000:5000"}, "image": "registry@" + aDigest}}}
|
||||
}
|
||||
|
||||
// **The trust a machine writes for the store, and the address every built image is fetched
|
||||
// through, say the port the node gave the store** — not the catalogue's number.
|
||||
func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
if _, err := assign(ctx, open, "anchor", "distribution"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the mesh built, recorded by digest and path, running on the other machine.
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "app", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "server", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app",
|
||||
"image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}})
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it
|
||||
// asks where this machine reaches the store, as the docker module does.
|
||||
register(t, open, catalogue.Manifest{Module: "runtime", Version: "1",
|
||||
Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json",
|
||||
"into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}})
|
||||
if _, err := assign(ctx, open, "laptop", "runtime"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
on := map[string]bool{"anchor": true, "laptop": true}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on)
|
||||
if err != nil || !found || node != "anchor" || port != "5101" {
|
||||
t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err)
|
||||
}
|
||||
|
||||
var trust string
|
||||
for _, r := range composed(t, open, "laptop").Resources {
|
||||
if r["path"] == "/etc/docker/daemon.json" {
|
||||
trust, _ = r["content"].(string)
|
||||
}
|
||||
if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Errorf("the image the mesh built is fetched as %v", r["image"])
|
||||
}
|
||||
}
|
||||
if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") {
|
||||
t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust)
|
||||
}
|
||||
|
||||
// And a replay to the catalogue says where the store is now.
|
||||
announced, err := following{open}.Announceable(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest {
|
||||
t.Fatalf("the replay announces %+v", announced)
|
||||
}
|
||||
}
|
||||
|
||||
// **The control plane's own connections say the port the node gave the store and the broker.**
|
||||
//
|
||||
// Composed from the control plane's own manifest against a real inventory: the store's module is
|
||||
// given 6852 on this node the way genesis or an operator gives it, and the control plane's
|
||||
// process is told so beside the sealed connection genesis wrote.
|
||||
func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
raw, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
||||
Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, control)
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}})
|
||||
register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh},
|
||||
{Port: 5672, From: catalogue.FromMesh}},
|
||||
Guards: []int{15672},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker",
|
||||
"ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}})
|
||||
// The control plane's own bus user is the installer's, seeded at genesis before the controller
|
||||
// runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203).
|
||||
if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller",
|
||||
Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The store's module is registered and given its port, and NOT assigned: the state genesis
|
||||
// leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085).
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "postgres",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var env map[string]any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "mesh-controller.controller" {
|
||||
env, _ = r["env"].(map[string]any)
|
||||
}
|
||||
}
|
||||
if env == nil {
|
||||
t.Fatal("the control plane's process is not in its own node's declaration")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "6852",
|
||||
"MESH_STORE_IDENTITY_PORT": "6852",
|
||||
"MESH_STORE_LICENCES_PORT": "6852",
|
||||
"MESH_BROKER_AMQP_PORT": "5679",
|
||||
// Neither given nor assigned by the mesh: the manifest's own number is NOT the answer,
|
||||
// because the sealed value beside it carries the port genesis wrote (finding F3).
|
||||
"MESH_BROKER_ADDRESS_PORT": "",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "",
|
||||
} {
|
||||
if env[key] != want {
|
||||
t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// withSeatPorts is the control plane's manifest with the seat placeholders in its environment —
|
||||
// added here until module.json carries them (the manifest lands one commit after the code that
|
||||
// fills it, so a control plane one build behind never sees a placeholder it cannot fill).
|
||||
func withSeatPorts(m catalogue.Manifest) catalogue.Manifest {
|
||||
seatPorts := map[string]string{
|
||||
"MESH_STORE_INVENTORY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_IDENTITY_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_STORE_LICENCES_PORT": "${seat:mesh-store:5432}",
|
||||
"MESH_BROKER_AMQP_PORT": "${seat:mesh-broker:5672}",
|
||||
"MESH_BROKER_MANAGEMENT_PORT": "${seat:mesh-broker:15672}",
|
||||
"MESH_BROKER_ADDRESS_PORT": "${seat:mesh-broker:5671}",
|
||||
}
|
||||
out := m
|
||||
out.Resources = nil
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] != "container" && r["type"] != "process" {
|
||||
out.Resources = append(out.Resources, r)
|
||||
continue
|
||||
}
|
||||
copied := map[string]any{}
|
||||
for k, v := range r {
|
||||
copied[k] = v
|
||||
}
|
||||
env := map[string]any{}
|
||||
if had, ok := r["env"].(map[string]any); ok {
|
||||
for k, v := range had {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
for k, v := range seatPorts {
|
||||
if _, said := env[k]; !said {
|
||||
env[k] = v
|
||||
}
|
||||
}
|
||||
copied["env"] = env
|
||||
out.Resources = append(out.Resources, copied)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// aLoneNode is one capable machine with nothing placed on any network — the control-node during
|
||||
// genesis, before the "network" step, which is after the store, the broker, the vault and the
|
||||
// catalogue have each been built and pushed (finding F2).
|
||||
func aLoneNode(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t)
|
||||
licences.ForTest(t)
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := open.inventory.AddNode(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}}})
|
||||
var profile map[string]any
|
||||
_ = json.Unmarshal(reported, &profile)
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// **Before the network exists, the store's own node reaches it by loopback** — never refused,
|
||||
// never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to
|
||||
// a node on no network, and builds the catalogue on a base it must be able to pull.
|
||||
func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) {
|
||||
open := aLoneNode(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aStore())
|
||||
register(t, open, catalogue.Manifest{Module: "builder", Version: "1",
|
||||
Requires: []string{catalogue.ArtifactStoreProvision},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder",
|
||||
"image": "registry.example/mesh-builder@" + aDigest}}})
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{
|
||||
ID: "b1", Repository: "r", Module: "postgres", Commit: "abc",
|
||||
Made: []inventory.Artifact{{Name: "runtime", Kind: "image",
|
||||
Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "postgres", Version: "1",
|
||||
Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres",
|
||||
"image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}})
|
||||
for _, module := range []string{"distribution", "builder", "postgres"} {
|
||||
if _, err := assign(ctx, open, "anchor", module); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "distribution",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var image any
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "postgres.runtime" {
|
||||
image = r["image"]
|
||||
}
|
||||
}
|
||||
if image != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image)
|
||||
}
|
||||
held := heldBy(ctx)
|
||||
if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest {
|
||||
t.Fatalf("a builder beside the store is handed the base %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,641 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: taking a module is its cutover; converging a node is one act, previewed, and
|
||||
// refused while a found container is held; returning to adopted keeps what was taken.
|
||||
|
||||
// anAdoptedAnchor is aMesh with the anchor adopted, running a served module the predecessor also
|
||||
// runs and a module with only a file, and a filter module in the catalogue.
|
||||
func anAdoptedAnchor(t *testing.T) (*stores, *[]string) {
|
||||
t.Helper()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 8080, From: catalogue.FromEverywhere}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hi"},
|
||||
{"id": "server", "type": "container", "name": "hello-web", "ports": []any{"8080:80"},
|
||||
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||
}})
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||
Resources: []map[string]any{
|
||||
{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"},
|
||||
}})
|
||||
register(t, open, catalogue.Manifest{Module: "nftables", Version: "1",
|
||||
Filtering: &catalogue.Filtering{Into: "/etc/nftables.conf"},
|
||||
Resources: []map[string]any{{"id": "load", "type": "service", "unit": "mesh-filter.service",
|
||||
"state": "running", "restart-on": []any{"filtering"}}}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, m := range []string{"hello-web", "notes"} {
|
||||
if _, err := assign(ctx, open, "anchor", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
sent := &[]string{}
|
||||
saved := sendNodes
|
||||
sendNodes = func(_ context.Context, _ *stores, names []string) error {
|
||||
*sent = append(*sent, names...)
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { sendNodes = saved })
|
||||
return open, sent
|
||||
}
|
||||
|
||||
// reportsHolding has the anchor report, on what it was last sent, holding what is given.
|
||||
func reportsHolding(t *testing.T, open *stores, held ...link.Held) {
|
||||
t.Helper()
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5000, By: "predecessor-registry",
|
||||
Published: true, ContainerPort: 5000},
|
||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker",
|
||||
Published: true, ContainerPort: 15672},
|
||||
}, held...)
|
||||
}
|
||||
|
||||
// reportsReaching has the anchor report, on what it was last sent, what is reachable on it and
|
||||
// holding what is given.
|
||||
func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ...link.Held) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
body, err := composed(t, open, "anchor").Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digestOf(body), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
|
||||
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
|
||||
Firewall: "ufw", Held: held, Reachable: reachable,
|
||||
// A machine says which of its links face outside on every apply (novox/hq ADR 0140), and a
|
||||
// filter is not sent to one that has not. The anchor reports one, as a real host does; this
|
||||
// fixture lacked it from 2026-09-28 and nothing ran the test (issue 177).
|
||||
Outward: []string{"eth0"},
|
||||
// And what filters it (ADR 0168): its front end, the runtime's own, and a chain a
|
||||
// predecessor left in the runtime's user chain.
|
||||
Filters: anchorFilters,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// anchorFilters is what the adopted anchor says filters it: ufw's chains, the runtime's, and a
|
||||
// predecessor's chain the mesh did not write.
|
||||
var anchorFilters = []link.Filter{
|
||||
{Where: "table ip filter, chain ufw-reject-input", Owner: "found-firewall", Refuses: "reject"},
|
||||
{Where: "table ip filter, chain DOCKER", Owner: "runtime", Refuses: `iifname != "docker0" oifname "docker0" drop`},
|
||||
{Where: "table ip filter, chain DOCKER-USER", Owner: "other", Refuses: `iifname "eth0" tcp dport 6000 drop`},
|
||||
}
|
||||
|
||||
var (
|
||||
heldContainer = link.Held{ID: "hello-web.server", Module: "hello-web", Kind: "container",
|
||||
Target: "hello-web", Since: time.Now()}
|
||||
heldFile = link.Held{ID: "notes.conf", Module: "notes", Kind: "file",
|
||||
Target: "/etc/notes.conf", Since: time.Now(), Kept: "/var/lib/mesh-host/kept/abc-notes.conf"}
|
||||
)
|
||||
|
||||
func TestTakingAModuleNotOnTheNodeIsRefused(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
if _, err := take(t.Context(), open, "anchor", "nftables", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAssigned) {
|
||||
t.Fatalf("taking an unassigned module gave %v", err)
|
||||
}
|
||||
if _, err := take(t.Context(), open, "laptop", "network", takeOptions{Yes: true}); !errors.Is(err, inventory.ErrNotAdopted) {
|
||||
t.Fatalf("taking on a converged node gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvergingIsRefusedOnAPreviewThatWouldBeStale(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
_, err := converge(t.Context(), open, "anchor", false, "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "has not reported") {
|
||||
t.Fatalf("a node that never reported was previewed: %v", err)
|
||||
}
|
||||
if len(*sent) != 0 {
|
||||
t.Fatal("a refused converge sent something")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFlipIsRefusedWhileAFoundContainerIsHeld(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
_, err := converge(t.Context(), open, "anchor", true, "", "")
|
||||
if err == nil || !strings.Contains(err.Error(), "take anchor hello-web once its data has moved") {
|
||||
t.Fatalf("the flip was not refused while hello-web holds its found container: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(t.Context(), "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTakingNamesWhatItReplaces(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
ctx := t.Context()
|
||||
// The machine holds something for the module, so the take acts on the preview the operator
|
||||
// saw and names its digest (novox/hq ADR 0163).
|
||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := takeDigestIn(t, preview)
|
||||
if !strings.Contains(preview, "nothing taken; `take anchor hello-web --yes "+saw+"`") {
|
||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||
}
|
||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||
t.Fatal("the preview took something")
|
||||
}
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err == nil || !strings.Contains(err.Error(), "name its digest") {
|
||||
t.Fatalf("--yes without the digest was not refused: %v", err)
|
||||
}
|
||||
said, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "container hello-web (hello-web.server)") ||
|
||||
!strings.Contains(said, "push anchor") {
|
||||
t.Fatalf("taking did not say what it replaces and what to run:\n%s", said)
|
||||
}
|
||||
}
|
||||
|
||||
// takeDigestIn is the digest a take's preview printed.
|
||||
func takeDigestIn(t *testing.T, preview string) string {
|
||||
t.Helper()
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||
return fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||
return ""
|
||||
}
|
||||
|
||||
// A take acts on the preview the operator saw, and on an account of the machine that is still the
|
||||
// machine: a changed preview and a stale account refuse (novox/hq ADR 0163, rule 1).
|
||||
func TestATakeIsRefusedOnAChangedPreviewOrAStaleAccount(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
reportsHolding(t, open, heldContainer, heldFile)
|
||||
preview, err := take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := takeDigestIn(t, preview)
|
||||
|
||||
// The machine reports again, and what it holds has changed: the found container now carries
|
||||
// facts the preview never showed.
|
||||
changed := heldContainer
|
||||
changed.Facts = map[string]any{"image": "hello:2", "declared_image": "registry.example/hello"}
|
||||
reportsHolding(t, open, changed, heldFile)
|
||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||
t.Fatalf("a changed preview was acted on: %v", err)
|
||||
}
|
||||
if taken, _ := open.inventory.Taken(ctx, "anchor"); len(taken) != 0 {
|
||||
t.Fatal("a refused take took something")
|
||||
}
|
||||
|
||||
// And an account older than the flip allows.
|
||||
preview, err = take(ctx, open, "anchor", "hello-web", takeOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw = takeDigestIn(t, preview)
|
||||
saved := reportFreshFor
|
||||
reportFreshFor = -time.Second
|
||||
defer func() { reportFreshFor = saved }()
|
||||
_, err = take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw})
|
||||
if err == nil || !strings.Contains(err.Error(), "a take acts only on an account newer than") {
|
||||
t.Fatalf("a stale account was acted on: %v", err)
|
||||
}
|
||||
reportFreshFor = saved
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true, Digest: saw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A module the machine holds nothing for has nothing to compare: --yes alone suffices.
|
||||
if _, err := take(ctx, open, "anchor", "notes", takeOptions{Yes: true}); err == nil {
|
||||
// notes holds a file, so this one needs the digest too.
|
||||
t.Fatal("notes holds a found file and was taken without a digest")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvergingPreviewsThenChangesAndAdoptingKeepsWhatWasTaken(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"tcp/8080 hello-web (published, container port 80)",
|
||||
"declared by hello-web (from anywhere)",
|
||||
"WILL CLOSE — no module assigned here declares it",
|
||||
// The anchor faces inward and is on the private network: the derived filter admits ssh
|
||||
// from the mesh only.
|
||||
"WILL CLOSE to everything outside the private network — ssh stays open from the mesh",
|
||||
"notes\n replacing the found file /etc/notes.conf (notes.conf), original kept at",
|
||||
"assigns nftables",
|
||||
"the found firewall (ufw) is disabled, never flushed",
|
||||
// What it routes is not a listener: said not to be previewed, and to be dropped.
|
||||
"not previewed: traffic the machine routes that is not a published port",
|
||||
"the derived filter drops it unless a module declares it",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
if strings.Contains(preview, "15672") {
|
||||
t.Errorf("a loopback listener is in the preview:\n%s", preview)
|
||||
}
|
||||
// What filters the machine now, and the fate of each (novox/hq ADR 0168): the predecessor's
|
||||
// chain is named as not the mesh's and left, so the reader knows before the flip.
|
||||
for _, want := range []string{
|
||||
"table ip filter, chain DOCKER-USER",
|
||||
"NOT THE MESH'S; left in force",
|
||||
`iifname "eth0" tcp dport 6000 drop`,
|
||||
"table ip filter, chain ufw-reject-input",
|
||||
"the found firewall's; retired with it",
|
||||
"the container runtime's own; left",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if strings.Contains(line, "5000") && !strings.Contains(line, "WILL CLOSE") {
|
||||
t.Errorf("an undeclared published port is not said to close: %s", line)
|
||||
}
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("the preview changed something")
|
||||
}
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, _ := open.inventory.NodeByName(ctx, "anchor")
|
||||
if n.Adopted {
|
||||
t.Fatal("converge --yes left the node adopted")
|
||||
}
|
||||
taken, _ := open.inventory.Taken(ctx, "anchor")
|
||||
if !reflect.DeepEqual(taken, []string{"hello-web", overlay.Name, "nftables", "notes"}) {
|
||||
t.Fatalf("the flip took %v", taken)
|
||||
}
|
||||
if !reflect.DeepEqual(*sent, []string{"anchor"}) {
|
||||
t.Fatalf("the flip sent %v", *sent)
|
||||
}
|
||||
declared := composed(t, open, "anchor")
|
||||
if declared.Adoption != nil {
|
||||
t.Fatal("a converged node is still sent an adoption envelope")
|
||||
}
|
||||
if !hasID(declared.Resources, "nftables.filtering") {
|
||||
t.Fatal("the converged node is not declared the mesh's filter")
|
||||
}
|
||||
|
||||
if _, err := adopt(ctx, open, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := adopt(ctx, open, "anchor"); err == nil {
|
||||
t.Fatal("adopting an adopted node was not refused")
|
||||
}
|
||||
again, _ := open.inventory.Taken(ctx, "anchor")
|
||||
if !reflect.DeepEqual(again, taken) {
|
||||
t.Fatalf("returning to adopted lost what was taken: %v", again)
|
||||
}
|
||||
declared = composed(t, open, "anchor")
|
||||
if declared.Adoption == nil || len(declared.Adoption.Untaken) != 0 {
|
||||
t.Fatalf("returned to adopted, the envelope is %+v", declared.Adoption)
|
||||
}
|
||||
if hasID(declared.Resources, "nftables.filtering") || hasID(declared.Resources, "nftables.load") {
|
||||
t.Fatal("returned to adopted, the mesh's filter is still declared")
|
||||
}
|
||||
}
|
||||
|
||||
func hasID(resources []map[string]any, id string) bool {
|
||||
for _, r := range resources {
|
||||
if r["id"] == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The command API refuses a flip exactly as the command line does, in the same words.
|
||||
func TestTheApiRefusesTheFlipInTheCommandLinesWords(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
reportsHolding(t, open, heldContainer)
|
||||
_, direct := converge(t.Context(), open, "anchor", true, "", "")
|
||||
if direct == nil {
|
||||
t.Fatal("the flip was not refused")
|
||||
}
|
||||
got := asking(t, letIn{}, "POST", "/converge", `{"node":"anchor","yes":true}`)
|
||||
if got.Code != http.StatusConflict {
|
||||
t.Fatalf("got %d: %s", got.Code, got.Body.String())
|
||||
}
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(got.Body.Bytes(), &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said["refused"] != direct.Error() {
|
||||
t.Fatalf("the API said %q and the command line %q", said["refused"], direct.Error())
|
||||
}
|
||||
if got := asking(t, letIn{}, "POST", "/take", `{"node":"anchor"}`); got.Code != http.StatusBadRequest {
|
||||
t.Fatalf("a take naming no module got %d", got.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the preview says what the derived filter does, rendered as it is rendered. On
|
||||
// a machine that faces inward, ssh is admitted from the private network only, and a port a module
|
||||
// admits from the mesh only closes to everything outside it: both are said to close.
|
||||
func TestThePreviewSaysWhatNarrowsToTheMeshCloses(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}})
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 5432, By: "postgres"},
|
||||
{Protocol: "tcp", Address: "10.77.0.1", Port: 5432, By: "postgres"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
})
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lines := map[string]string{}
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 1 && strings.HasPrefix(fields[0], "tcp/") {
|
||||
lines[fields[0]+" "+fields[1]] += line + "\n"
|
||||
}
|
||||
}
|
||||
if got := lines["tcp/22 sshd"]; !strings.Contains(got, "WILL CLOSE to everything outside "+
|
||||
"the private network") {
|
||||
t.Errorf("ssh on an inward machine is not said to close outside the mesh:\n%s", preview)
|
||||
}
|
||||
store := lines["tcp/5432 postgres"]
|
||||
if strings.Count(store, "WILL CLOSE to everything outside the private network") != 1 ||
|
||||
!strings.Contains(store, "declared by store (from mesh)") {
|
||||
t.Errorf("the store's narrowing is not said to close, or its mesh address is:\n%s", preview)
|
||||
}
|
||||
if got := lines["tcp/8080 hello-web"]; !strings.Contains(got, "declared by hello-web (from anywhere)") {
|
||||
t.Errorf("a port open to everywhere is not said to stay:\n%s", preview)
|
||||
}
|
||||
}
|
||||
|
||||
// digestIn is the digest a converge preview printed.
|
||||
func digestIn(t *testing.T, preview string) string {
|
||||
t.Helper()
|
||||
for _, line := range strings.Split(preview, "\n") {
|
||||
if fields := strings.Fields(line); len(fields) == 2 && fields[0] == "preview" {
|
||||
return fields[1]
|
||||
}
|
||||
}
|
||||
t.Fatalf("the preview printed no digest:\n%s", preview)
|
||||
return ""
|
||||
}
|
||||
|
||||
// The flip acts on the preview the operator saw: it names that preview's digest, and it is refused
|
||||
// when the digest is missing, when anything the preview says has changed since, or when the node's
|
||||
// account of itself is too old to be the machine as it is.
|
||||
func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saw := digestIn(t, preview)
|
||||
if !strings.Contains(preview, "converge anchor --yes "+saw) {
|
||||
t.Fatalf("the preview does not say how to act on it:\n%s", preview)
|
||||
}
|
||||
unchanged := func() {
|
||||
t.Helper()
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, "", ""); err == nil ||
|
||||
!strings.Contains(err.Error(), "--yes "+saw) {
|
||||
t.Fatalf("a flip naming no preview was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
// Something new is reachable: the preview the operator saw is not what would happen.
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 8080, By: "hello-web", Published: true,
|
||||
ContainerPort: 80},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 6000, By: "something-new"},
|
||||
}, heldFile)
|
||||
_, err = converge(ctx, open, "anchor", true, saw, "")
|
||||
if err == nil || !strings.Contains(err.Error(), "has changed since preview "+saw) {
|
||||
t.Fatalf("a flip on a changed preview was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
// An account older than the flip trusts is refused, whatever digest is named.
|
||||
again, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := reportFreshFor
|
||||
reportFreshFor = time.Nanosecond
|
||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, again), "")
|
||||
reportFreshFor = saved
|
||||
if err == nil || !strings.Contains(err.Error(), "wait for its next report") {
|
||||
t.Fatalf("a flip on an old account was not refused: %v", err)
|
||||
}
|
||||
unchanged()
|
||||
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, again), ""); err != nil {
|
||||
t.Fatalf("the flip on the preview just seen was refused: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); n.Adopted {
|
||||
t.Fatal("the flip did not converge the node")
|
||||
}
|
||||
}
|
||||
|
||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||
// flip is not made to wait on the flip's own hold.
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var heldElsewhere, heldHere error
|
||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||
// Another caller cannot hold the node while the flip sends it.
|
||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||
release()
|
||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||
}
|
||||
// The flip's own send holds it without waiting on itself.
|
||||
_, release, err := holdNodes(inner, open, names)
|
||||
if err != nil {
|
||||
heldHere = err
|
||||
return err
|
||||
}
|
||||
release()
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heldElsewhere != nil || heldHere != nil {
|
||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||
}
|
||||
// And given back once it is done.
|
||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||
// not only its files, and the digest changes when any of them does.
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Now()
|
||||
held := []link.Held{heldFile,
|
||||
{ID: "notes.data", Module: "notes", Kind: "directory", Target: "/var/lib/notes", Since: since},
|
||||
{ID: "notes.daemon", Module: "notes", Kind: "service", Target: "notes.service", Since: since},
|
||||
{ID: "notes.seed", Module: "notes", Kind: "archive", Target: "/srv/notes", Since: since},
|
||||
{ID: "notes.worker", Module: "notes", Kind: "process", Target: "notes-worker", Since: since},
|
||||
{ID: "notes.account", Module: "notes", Kind: "user", Target: "notes", Since: since},
|
||||
}
|
||||
reportsHolding(t, open, held...)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"replacing the found directory /var/lib/notes (notes.data)",
|
||||
"replacing the found service notes.service (notes.daemon)",
|
||||
"replacing the found archive /srv/notes (notes.seed)",
|
||||
"replacing the found process notes-worker (notes.worker)",
|
||||
"replacing the found user notes (notes.account)",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview does not say %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
reportsHolding(t, open, held[:len(held)-1]...)
|
||||
fewer, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if digestIn(t, fewer) == digestIn(t, preview) {
|
||||
t.Fatal("the digest does not change with what is held")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the flip acts on what the node said is reachable, so an account naming nothing
|
||||
// is refused. Every machine that is up answers on ssh; nothing reported means the host's collectors
|
||||
// did not, and flipping would close ports the preview never named.
|
||||
func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
open, sent := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only a loopback listener: nothing off the machine, which is the same silence.
|
||||
reportsReaching(t, open, []link.Reach{
|
||||
{Protocol: "tcp", Address: "127.0.0.1", Port: 15672, By: "mesh-broker"},
|
||||
}, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(preview, "this account looks partial") {
|
||||
t.Errorf("the preview does not mark a partial account:\n%s", preview)
|
||||
}
|
||||
_, err = converge(ctx, open, "anchor", true, digestIn(t, preview), "")
|
||||
if err == nil || !strings.Contains(err.Error(), "says nothing is reachable on it") {
|
||||
t.Fatalf("the flip was not refused on an account naming nothing: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "anchor"); !n.Adopted || len(*sent) != 0 {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||
// holds the node, so it waits for whatever is converging it.
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||
|
||||
var whileFlipping error
|
||||
sendNodes = func(context.Context, *stores, []string) error {
|
||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,186 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The command API: what the mesh can be asked to do, over a network.
|
||||
//
|
||||
// **An adapter and nothing else** (novox/hq ADR 0035). Every route here calls the same function
|
||||
// the command line calls, so a refusal is the same refusal in the same words. Nothing is decided
|
||||
// in this file — the moment it validates something the command line does not, the mesh has two
|
||||
// answers to one question.
|
||||
//
|
||||
// **It refuses everything unless it was told how to know who is asking.** The board is published
|
||||
// on a public name, and this is what stands behind it: an unauthenticated command surface reachable
|
||||
// from the internet is authority over the mesh handed to whoever finds it. So there is no
|
||||
// permissive default and no flag that removes the check — a mesh that has not been told how to
|
||||
// authenticate serves nothing, loudly.
|
||||
//
|
||||
// The intended authenticator is an OAuth2 provider (ADR 0035), which is an ordinary module. Until
|
||||
// one is configured this refuses, which is the correct behaviour rather than a placeholder: a
|
||||
// surface that worked without authentication would be one somebody left running.
|
||||
func apiCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("api", flag.ContinueOnError)
|
||||
listen := set.String("listen", "127.0.0.1:8081", "where to serve it")
|
||||
issuer := set.String("issuer", "",
|
||||
"the OAuth2 issuer whose tokens this accepts; without it, nothing is served")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.TrimSpace(*issuer) == "" {
|
||||
// Refused at start rather than per request, so it is discovered by whoever ran it rather
|
||||
// than by whoever finds it.
|
||||
return errors.New(
|
||||
"--issuer is not set, and this serves commands rather than pages: it will not run " +
|
||||
"without being told whose tokens to believe. An OAuth2 provider is an ordinary " +
|
||||
"module (novox/hq ADR 0035)")
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: *listen,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
Handler: commands(mustAuthenticate(*issuer)),
|
||||
}
|
||||
fmt.Printf("the command API is on http://%s\n", *listen)
|
||||
fmt.Printf(" it accepts tokens from %s and refuses everything else\n", *issuer)
|
||||
fmt.Printf(" every route calls what the command line calls\n")
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(closing)
|
||||
}()
|
||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Authenticator says whether a request may act, and as whom.
|
||||
//
|
||||
// An interface so the check can be driven by a test without an identity provider, and so the one
|
||||
// real implementation is the only thing that has to be right.
|
||||
type Authenticator interface {
|
||||
// Who returns the subject a request is acting as, or an error naming why it may not.
|
||||
Who(r *http.Request) (string, error)
|
||||
}
|
||||
|
||||
// mustAuthenticate is the real one: a bearer token from the configured issuer.
|
||||
//
|
||||
// **Not yet verifying the signature**, and it says so rather than pretending. Verification needs
|
||||
// the issuer's keys, which needs an identity provider to exist — so this refuses every request
|
||||
// until that is built, which is the same answer as having no API at all and is honest about why.
|
||||
func mustAuthenticate(issuer string) Authenticator { return notYet{issuer: issuer} }
|
||||
|
||||
type notYet struct{ issuer string }
|
||||
|
||||
func (n notYet) Who(*http.Request) (string, error) {
|
||||
return "", fmt.Errorf(
|
||||
"this mesh has no way to verify a token from %s yet: the identity provider is a module "+
|
||||
"and none is running. Use the command line, which authenticates through nothing "+
|
||||
"because it is already behind the machine's own login", n.issuer)
|
||||
}
|
||||
|
||||
// commands is the routing, separate so a test can drive it without a listener.
|
||||
func commands(who Authenticator) http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("POST /assign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return assign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
mux.HandleFunc("POST /unassign", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return unassign(ctx, open, in.Node, splitModules(in.Module)...)
|
||||
}))
|
||||
// Adoption (novox/hq ADR 0100): the same acts as `take`, `converge` and `adopt`.
|
||||
mux.HandleFunc("POST /take", acting(who, true, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return take(ctx, open, in.Node, in.Module, takeOptions{Yes: in.Yes, Digest: in.Digest})
|
||||
}))
|
||||
mux.HandleFunc("POST /converge", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return converge(ctx, open, in.Node, in.Yes, in.Digest, in.Filter)
|
||||
}))
|
||||
mux.HandleFunc("POST /adopt", acting(who, false, func(ctx context.Context, open *stores, in request) (string, error) {
|
||||
return adopt(ctx, open, in.Node)
|
||||
}))
|
||||
|
||||
// Anything else is said plainly, because a command surface answering 404 to a verb somebody
|
||||
// expected is indistinguishable from one that is down.
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
refuse(w, http.StatusNotFound, fmt.Errorf(
|
||||
"%s %s is not something this mesh can be asked; it accepts POST /assign, "+
|
||||
"POST /unassign, POST /take, POST /converge and POST /adopt", r.Method, r.URL.Path))
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
// Yes, Digest and Filter are converge's and take's: do it rather than preview it, the digest
|
||||
// of the preview it acts on, and (converge) which module loads the mesh's filter.
|
||||
Yes bool `json:"yes,omitempty"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
Filter string `json:"filter,omitempty"`
|
||||
}
|
||||
|
||||
// acting is the shape every route shares: authenticate, read, act, answer.
|
||||
func acting(
|
||||
who Authenticator,
|
||||
needsModule bool,
|
||||
do func(context.Context, *stores, request) (string, error),
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, err := who.Who(r); err != nil {
|
||||
refuse(w, http.StatusUnauthorized, err)
|
||||
return
|
||||
}
|
||||
var in request
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
refuse(w, http.StatusBadRequest, fmt.Errorf("this is not a request this understands: %w", err))
|
||||
return
|
||||
}
|
||||
if in.Node == "" || (needsModule && in.Module == "") {
|
||||
if needsModule {
|
||||
refuse(w, http.StatusBadRequest, errors.New(`both "node" and "module" are needed`))
|
||||
} else {
|
||||
refuse(w, http.StatusBadRequest, errors.New(`"node" is needed`))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
open, err := openStores(r.Context())
|
||||
if err != nil {
|
||||
refuse(w, http.StatusServiceUnavailable, err)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
said, err := do(r.Context(), open, in)
|
||||
if err != nil {
|
||||
// **The refusal the command line would have given, unchanged.** Carrying `said` with
|
||||
// it matters: an assignment that was kept and still does not resolve is two facts,
|
||||
// and dropping either makes the answer wrong.
|
||||
answer(w, http.StatusConflict, map[string]any{"said": said, "refused": err.Error()})
|
||||
return
|
||||
}
|
||||
answer(w, http.StatusOK, map[string]any{"said": said})
|
||||
}
|
||||
}
|
||||
|
||||
func answer(w http.ResponseWriter, status int, body map[string]any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(body)
|
||||
}
|
||||
|
||||
func refuse(w http.ResponseWriter, status int, err error) {
|
||||
answer(w, status, map[string]any{"refused": err.Error()})
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type letIn struct{}
|
||||
|
||||
func (letIn) Who(*http.Request) (string, error) { return "somebody", nil }
|
||||
|
||||
func asking(t *testing.T, who Authenticator, method, path, body string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
recorded := httptest.NewRecorder()
|
||||
commands(who).ServeHTTP(recorded, httptest.NewRequest(method, path, strings.NewReader(body)))
|
||||
return recorded
|
||||
}
|
||||
|
||||
// **Nothing is served to somebody the mesh cannot identify**, and that is the default rather than
|
||||
// a setting. The board this stands behind is published on a public name (novox/hq 11-a-board), so
|
||||
// an unauthenticated command surface is authority over the mesh handed to whoever finds it.
|
||||
func TestAnUnauthenticatedRequestIsRefused(t *testing.T) {
|
||||
got := asking(t, mustAuthenticate("https://identity.example/realms/mesh"),
|
||||
"POST", "/assign", `{"node":"anchor","module":"umami"}`)
|
||||
if got.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("an unidentified caller got %d", got.Code)
|
||||
}
|
||||
// And it says what to do instead, because the answer is not "give up".
|
||||
if !strings.Contains(got.Body.String(), "command line") {
|
||||
t.Errorf("the refusal does not say what still works: %s", got.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The API refuses to start at all without being told whose tokens to believe.
|
||||
//
|
||||
// At start rather than per request, so it is found by whoever ran it rather than by whoever
|
||||
// finds it.
|
||||
func TestTheApiWillNotRunWithoutAnIssuer(t *testing.T) {
|
||||
err := apiCommand(context.Background(), []string{})
|
||||
if err == nil {
|
||||
t.Fatal("it served commands without being told who may give them")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "issuer") {
|
||||
t.Errorf("the refusal does not name what is missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A request missing what it acts on is refused before anything is opened.
|
||||
func TestARequestThatNamesNothingIsRefused(t *testing.T) {
|
||||
// **The exact refusal, not merely "not accepted".** Asserting non-200 passes even when the
|
||||
// request got as far as opening a store and failing there, which proves nothing about whether
|
||||
// anything was checked — that is what the first version of this test did.
|
||||
for _, body := range []string{`{}`, `{"node":"anchor"}`, `{"module":"umami"}`, `not json`} {
|
||||
got := asking(t, letIn{}, "POST", "/assign", body)
|
||||
if got.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s got %d, and a request naming nothing is a bad request", body, got.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A verb nobody implemented is said plainly. A command surface answering 404 to something
|
||||
// somebody expected is indistinguishable from one that is down.
|
||||
func TestAnUnknownRouteSaysWhatIsAccepted(t *testing.T) {
|
||||
got := asking(t, letIn{}, "POST", "/rotate", `{}`)
|
||||
if got.Code != http.StatusNotFound {
|
||||
t.Fatalf("got %d", got.Code)
|
||||
}
|
||||
var said map[string]any
|
||||
_ = json.Unmarshal(got.Body.Bytes(), &said)
|
||||
if !strings.Contains(said["refused"].(string), "/assign") {
|
||||
t.Errorf("it does not say what it does accept: %v", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// ask calls one of a module's tools, through the control plane's own broker connection.
|
||||
//
|
||||
// A module serves tools under an account scoped to exactly that (novox/hq ADR 0047), and nothing
|
||||
// else in the mesh held an account that could ask one — not an operator at a terminal, not an agent
|
||||
// acting for one (novox/hq 04-ISSUES/049). The control plane does, so it is the way in: one
|
||||
// process, one connection, one place a question can be seen to have been asked (ADR 0095).
|
||||
func askCommand(ctx context.Context, args []string) error {
|
||||
positionals, flags := split(args)
|
||||
set := flag.NewFlagSet("ask", flag.ContinueOnError)
|
||||
wait := set.Duration("wait", 60*time.Second, "how long to wait for the module's answer")
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) < 2 || len(positionals) > 3 {
|
||||
return errors.New("ask <module> <tool> [json arguments] [--wait 60s]")
|
||||
}
|
||||
module, tool := positionals[0], positionals[1]
|
||||
var arguments json.RawMessage
|
||||
if len(positionals) == 3 {
|
||||
if !json.Valid([]byte(positionals[2])) {
|
||||
return fmt.Errorf("the arguments are not JSON: %s", positionals[2])
|
||||
}
|
||||
arguments = json.RawMessage(positionals[2])
|
||||
}
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
answer, err := link.Ask(ctx, server.Bus(), module, tool, arguments, *wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The answer as the module gave it, to standard output, for a person or a program. A tool
|
||||
// that answered with an error has still answered: printed the same way, and the exit status
|
||||
// says which.
|
||||
body, err := json.Marshal(answer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, string(body))
|
||||
if answer.Error != "" {
|
||||
return fmt.Errorf("%s.%s answered with an error: %s", module, tool, answer.Error)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// boardCommand serves the three questions as a page.
|
||||
//
|
||||
// **It reads through the same functions everything else does and holds nothing**
|
||||
// (novox/hq 03-DESIGN/01-to-be/11-a-board.md). The board being replaced is one service that reads
|
||||
// every context's database directly — [ADR 0008](novox/hq) violated by the one component with a
|
||||
// reason to violate it, and the cost is that a boundary nothing may cross can move, while one
|
||||
// thing crossing it is enough to freeze it. A board that reads the provisioning tables is a board
|
||||
// that breaks when provisioning changes its tables, and the change then gets weighed against the
|
||||
// board.
|
||||
//
|
||||
// **It stores nothing of its own.** No cache that can disagree, no table of what the mesh looked
|
||||
// like last time. Every request reads the mesh now; if that is slow, the answer belongs in the
|
||||
// context that owns it, where everything else asking gets it too.
|
||||
//
|
||||
// **Reading is the whole of it.** Every action a board could offer already exists as a command,
|
||||
// and a button that does something no command does is a second implementation of a decision.
|
||||
func boardCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("board", flag.ContinueOnError)
|
||||
// The private network, not everything. A board says which machines are broken and what they
|
||||
// are running, which is exactly the map somebody attacking this would like — and there is no
|
||||
// reason for it to be reachable from further away than the mesh.
|
||||
listen := set.String("listen", "127.0.0.1:8080", "where to serve it")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: *listen,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
Handler: board(),
|
||||
}
|
||||
fmt.Printf("the board is on http://%s\n", *listen)
|
||||
fmt.Printf(" it reads the mesh on every request and keeps nothing\n")
|
||||
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
closing, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = server.Shutdown(closing)
|
||||
}()
|
||||
if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// board is the handler, separate so a test can drive it without a listener.
|
||||
func board() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
asked, err := ask(r.Context())
|
||||
if err != nil {
|
||||
// **Said, not blank.** A board that cannot reach the mesh and renders an empty page
|
||||
// says "nothing is wrong" in the one situation where nobody can know that.
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
_ = page.Execute(w, view{Unreachable: err.Error()})
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := page.Execute(w, asked); err != nil {
|
||||
// The page is half-written by now; there is nothing useful left to say to the
|
||||
// browser, and saying it here is what stops the failure being silent.
|
||||
fmt.Printf("the board could not render: %v\n", err)
|
||||
}
|
||||
})
|
||||
// The same answers for something that is not a person, from the same read. A board and a
|
||||
// script disagreeing about which machine is broken would be worse than either alone.
|
||||
mux.HandleFunc("/mesh.json", func(w http.ResponseWriter, r *http.Request) {
|
||||
open, err := openStores(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
defer open.Close()
|
||||
asked, err := theThreeQuestions(r.Context(), open)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write(append(body, '\n'))
|
||||
})
|
||||
return mux
|
||||
}
|
||||
|
||||
// ask reads the mesh for one request.
|
||||
func ask(ctx context.Context) (view, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return view{}, err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return view{}, err
|
||||
}
|
||||
return viewOf(asked), nil
|
||||
}
|
||||
|
||||
// view is what the page is given. Nothing is derived here that the reader could not derive.
|
||||
type view struct {
|
||||
Unreachable string
|
||||
Machines int
|
||||
Broken []brokenMachine
|
||||
Quiet []quietMachine
|
||||
Behind []staleModule
|
||||
Waiting []waitingMachine
|
||||
// Unresolved is every machine that cannot be worked out at all. Shown above everything else,
|
||||
// because a machine here is in none of the other lists: nothing was computed for it, so it is
|
||||
// not broken, not quiet and not behind — and a page without this said "all well" about a mesh
|
||||
// where nothing could be sent anywhere.
|
||||
Unresolved []blockedMachine
|
||||
// Network is why the private network could not be computed, when it could not.
|
||||
Network string
|
||||
// Adopted is every node still adopted (novox/hq ADR 0100). Not broken: nothing forces the
|
||||
// flip, so a node left adopted is shown rather than read as converged.
|
||||
Adopted []string
|
||||
At string
|
||||
}
|
||||
|
||||
type blockedMachine struct {
|
||||
Node string
|
||||
// Said is the mesh's own words, a line at a time. Every unmet requirement, not the first:
|
||||
// a machine is usually blocked by more than one and fixing one of them changes nothing.
|
||||
Said []string
|
||||
}
|
||||
|
||||
type waitingMachine struct {
|
||||
Node string
|
||||
// Never told is not out of date: nobody has ever asked this machine to be anything. Same
|
||||
// remedy, different situation, and the page says which.
|
||||
Never bool
|
||||
}
|
||||
|
||||
type brokenMachine struct {
|
||||
Node string
|
||||
// Outcome is refused or failed, and stays distinct all the way to the page. **Refused means
|
||||
// the machine is exactly as it was and what is wrong is in what was sent; failed means it is
|
||||
// in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
||||
// different places, so one word for both would send half the readers to the wrong one.
|
||||
Outcome string
|
||||
Said []string
|
||||
When string
|
||||
}
|
||||
|
||||
type quietMachine struct {
|
||||
Node string
|
||||
// Heard is "never" or how long ago. Never heard from is not the same as quiet for a while:
|
||||
// one may be a machine that was never sent anything.
|
||||
Heard string
|
||||
}
|
||||
|
||||
type staleModule struct {
|
||||
Module string
|
||||
Holds string
|
||||
Source string
|
||||
Running []string
|
||||
}
|
||||
|
||||
func viewOf(asked answers) view {
|
||||
out := view{Machines: len(asked.nodes), At: time.Now().Format("15:04:05"),
|
||||
Network: asked.network, Adopted: adoptedNodes(asked.nodes)}
|
||||
var blocked []string
|
||||
for name := range asked.refused {
|
||||
blocked = append(blocked, name)
|
||||
}
|
||||
sort.Strings(blocked)
|
||||
for _, name := range blocked {
|
||||
one := blockedMachine{Node: name}
|
||||
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
||||
one.Said = append(one.Said, strings.TrimSpace(line))
|
||||
}
|
||||
out.Unresolved = append(out.Unresolved, one)
|
||||
}
|
||||
for _, d := range asked.wrong {
|
||||
one := brokenMachine{Node: d.Node, Outcome: d.Outcome,
|
||||
When: d.At.Local().Format("2006-01-02 15:04")}
|
||||
if d.Refused != "" {
|
||||
// The host's own words. It says exactly what it could not accept, and nothing
|
||||
// written here would say it better.
|
||||
one.Said = append(one.Said, firstLine(d.Refused))
|
||||
}
|
||||
for _, f := range d.Failed {
|
||||
one.Said = append(one.Said, f.ID+": "+firstLine(f.Error))
|
||||
}
|
||||
out.Broken = append(out.Broken, one)
|
||||
}
|
||||
for _, n := range asked.quiet {
|
||||
out.Quiet = append(out.Quiet, quietMachine{Node: n.Name, Heard: heardFrom(n)})
|
||||
}
|
||||
for _, m := range asked.waiting {
|
||||
out.Waiting = append(out.Waiting, waitingMachine{Node: m.Node, Never: m.Never})
|
||||
}
|
||||
for module, on := range asked.behind {
|
||||
from := asked.sources[module]
|
||||
out.Behind = append(out.Behind, staleModule{
|
||||
Module: module, Holds: short(from.BuiltFrom), Source: short(from.Head), Running: on,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The page. Deliberately one file with no assets: a board that cannot render without fetching
|
||||
// something is a board that is blank exactly when the mesh is unwell.
|
||||
var page = template.Must(template.New("board").Parse(`<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>the mesh</title>
|
||||
<style>
|
||||
:root { color-scheme: light dark; }
|
||||
body { font: 15px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; margin: 2rem auto;
|
||||
max-width: 52rem; padding: 0 1rem; }
|
||||
h1 { font-size: 1.1rem; font-weight: 600; margin: 0 0 1.5rem; }
|
||||
h2 { font-size: 1rem; font-weight: 600; margin: 2rem 0 .5rem; }
|
||||
.quiet { opacity: .65; }
|
||||
.said { opacity: .8; padding-left: 1.5rem; }
|
||||
.outcome { display: inline-block; min-width: 4.5rem; }
|
||||
.refused { color: #b26b00; }
|
||||
.failed { color: #c0392b; }
|
||||
ul { list-style: none; padding: 0; margin: 0; }
|
||||
li { padding: .15rem 0; }
|
||||
footer { margin-top: 3rem; opacity: .6; font-size: .85rem; }
|
||||
</style>
|
||||
{{if .Unreachable}}
|
||||
<h1>the mesh cannot be read</h1>
|
||||
<p class="failed">{{.Unreachable}}</p>
|
||||
<p class="quiet">This says nothing about whether the mesh is well — only that this page could not
|
||||
find out.</p>
|
||||
{{else}}
|
||||
<h1>{{.Machines}} machine{{if ne .Machines 1}}s{{end}}</h1>
|
||||
|
||||
{{if .Unresolved}}
|
||||
<h2>Can everything be worked out?</h2>
|
||||
<ul>
|
||||
{{range .Unresolved}}
|
||||
<li><span class="outcome failed">blocked</span> <strong>{{.Node}}</strong>
|
||||
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
<p class="quiet">Nothing can be sent to a machine here, and it appears in none of the lists below:
|
||||
nothing was computed for it, so there is nothing it can be behind.</p>
|
||||
{{end}}
|
||||
{{if .Network}}
|
||||
<p class="failed">The private network could not be computed: {{.Network}}</p>
|
||||
{{end}}
|
||||
|
||||
<h2>Is anything broken?</h2>
|
||||
{{if .Broken}}
|
||||
<ul>
|
||||
{{range .Broken}}
|
||||
<li>
|
||||
<span class="outcome {{.Outcome}}">{{.Outcome}}</span>
|
||||
<strong>{{.Node}}</strong> <span class="quiet">{{.When}}</span>
|
||||
{{range .Said}}<div class="said">{{.}}</div>{{end}}
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
{{else}}<p class="quiet">No. Every machine is doing what it was told.</p>{{end}}
|
||||
|
||||
<h2>Is anything not answering?</h2>
|
||||
{{if .Quiet}}
|
||||
<ul>{{range .Quiet}}<li><strong>{{.Node}}</strong> <span class="quiet">{{.Heard}}</span></li>{{end}}</ul>
|
||||
<p class="quiet">Not heard from is not the same as tried and could not — a machine here may be
|
||||
new, switched off, or unreachable.</p>
|
||||
{{else}}<p class="quiet">No. Every machine has been heard from.</p>{{end}}
|
||||
|
||||
<h2>Is anything out of date?</h2>
|
||||
{{if .Behind}}
|
||||
<ul>
|
||||
{{range .Behind}}
|
||||
<li><strong>{{.Module}}</strong> holds {{.Holds}}, source has {{.Source}}
|
||||
{{if .Running}}<div class="said">running on {{range $i, $n := .Running}}{{if $i}}, {{end}}{{$n}}{{end}}</div>
|
||||
{{else}}<div class="said quiet">assigned to nothing</div>{{end}}
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
{{else}}<p class="quiet">No. Every module is what its source last had.</p>{{end}}
|
||||
|
||||
{{if .Waiting}}
|
||||
<ul>
|
||||
{{range .Waiting}}
|
||||
<li><strong>{{.Node}}</strong>
|
||||
{{if .Never}}<span class="quiet">has never been sent anything</span>
|
||||
{{else}}<span class="quiet">is not running what the mesh would send it</span>{{end}}
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
<p class="quiet">Never told is not out of date: nobody has asked that machine to be anything yet.
|
||||
Both are sent by <code>push --behind</code>.</p>
|
||||
{{else}}<p class="quiet">Every machine is running what the mesh would send it.</p>{{end}}
|
||||
{{if .Adopted}}
|
||||
<h2>Which machines are adopted?</h2>
|
||||
<ul>{{range .Adopted}}<li><strong>{{.}}</strong> <span class="quiet">adopted — what was found on it is kept until each module is taken</span></li>{{end}}</ul>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
<footer>Read at {{.At}}. This page holds nothing and changes nothing.</footer>
|
||||
`))
|
||||
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Refused and failed stay distinct all the way to the page.
|
||||
//
|
||||
// **Refused means the machine is exactly as it was and what is wrong is in what was sent; failed
|
||||
// means it is in a state nobody declared and what is wrong is on the machine.** They are fixed in
|
||||
// different places, so a page saying "error" for both sends half its readers to the wrong one.
|
||||
func TestRefusedAndFailedReachThePageAsDifferentThings(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{
|
||||
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
wrong: []inventory.Doing{
|
||||
{Node: "anchor", Outcome: "refused", Refused: "not a declaration this host speaks",
|
||||
At: time.Now()},
|
||||
{Node: "laptop", Outcome: "failed", At: time.Now(),
|
||||
Failed: []inventory.FailedResource{{ID: "web.container", Error: "no such image"}}},
|
||||
},
|
||||
}))
|
||||
// The rendered outcome, not the word anywhere on the page: both words appear in the
|
||||
// stylesheet, so a plain Contains passes whatever the machine actually said. It did.
|
||||
for _, want := range []string{`<span class="outcome refused">refused</span>`,
|
||||
`<span class="outcome failed">failed</span>`} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Fatalf("the page does not distinguish the two — missing %s:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
// And the host's own words, which say exactly what it could not accept.
|
||||
for _, want := range []string{"not a declaration this host speaks", "web.container", "no such image"} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Fatalf("the page does not say %q, so a reader must go and ask:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing wrong is said, not left blank. An empty page and a well mesh must not look alike.
|
||||
func TestAWellMeshSaysSoRatherThanShowingNothing(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
||||
for _, want := range []string{
|
||||
"Every machine is doing what it was told",
|
||||
"Every machine has been heard from",
|
||||
"Every module is what its source last had",
|
||||
} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Fatalf("a well mesh does not say %q:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A board that cannot reach the mesh must not render an empty page: that says "nothing is wrong"
|
||||
// in the one situation where nobody can know it.
|
||||
func TestABoardThatCannotReadTheMeshSaysSo(t *testing.T) {
|
||||
rendered := render(t, view{Unreachable: "the store did not answer"})
|
||||
if !strings.Contains(rendered, "the store did not answer") {
|
||||
t.Fatalf("the reason is not on the page:\n%s", rendered)
|
||||
}
|
||||
if strings.Contains(rendered, "Every machine is doing what it was told") {
|
||||
t.Fatal("a board that could not read the mesh reported that the mesh is well")
|
||||
}
|
||||
}
|
||||
|
||||
// Quiet is not broken, and the page says which it is.
|
||||
func TestQuietIsNotReportedAsBroken(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{
|
||||
nodes: []inventory.Node{{Name: "laptop"}},
|
||||
quiet: []inventory.Node{{Name: "laptop"}},
|
||||
}))
|
||||
if !strings.Contains(rendered, "not the same as tried and could not") {
|
||||
t.Fatalf("the page does not separate quiet from broken:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "Every machine is doing what it was told") {
|
||||
t.Fatalf("a quiet machine was counted as broken:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
// The page renders what a machine said, and a hostile string in it is not markup.
|
||||
//
|
||||
// What is on this page comes from machines, and a machine's own words are the whole reason the
|
||||
// page is useful. They are also the one thing here that nobody in this repository wrote.
|
||||
func TestWhatAMachineSaidIsNotMarkup(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{
|
||||
nodes: []inventory.Node{{Name: "anchor"}},
|
||||
wrong: []inventory.Doing{{Node: "anchor", Outcome: "refused",
|
||||
Refused: `<script>alert("from the machine")</script>`, At: time.Now()}},
|
||||
}))
|
||||
if strings.Contains(rendered, "<script>alert") {
|
||||
t.Fatalf("a machine's words were rendered as markup:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "<script>") {
|
||||
t.Fatalf("the words were not shown at all, so the reader cannot see what it said:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
// A module behind its source names the machines running the old one — the part with consequences.
|
||||
func TestBeingBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{
|
||||
nodes: []inventory.Node{{Name: "laptop"}},
|
||||
behind: map[string][]string{"web": {"laptop"}},
|
||||
sources: map[string]inventory.Source{"web": {BuiltFrom: "aaaaaaaaaa", Head: "bbbbbbbbbb"}},
|
||||
}))
|
||||
for _, want := range []string{"web", "aaaaaaaa", "bbbbbbbb", "running on laptop"} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Fatalf("the page does not say %q:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func render(t *testing.T, v view) string {
|
||||
t.Helper()
|
||||
var out strings.Builder
|
||||
if err := page.Execute(&out, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// The board and the command must answer the same question the same way.
|
||||
//
|
||||
// A machine not running what the mesh would send it was added to `status` and would have been
|
||||
// missing here — which is the one thing this page's design forbids: two answers to the same
|
||||
// question, disagreeing, with a person in front of each.
|
||||
func TestTheBoardSaysWhichMachinesHaveNotBeenSentWhatTheyShouldBe(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{
|
||||
nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
waiting: []inventory.Machine{
|
||||
{Node: "anchor"},
|
||||
{Node: "laptop", Never: true},
|
||||
},
|
||||
}))
|
||||
if !strings.Contains(rendered, "is not running what the mesh would send it") {
|
||||
t.Fatalf("the page does not say a machine is out of date:\n%s", rendered)
|
||||
}
|
||||
// Never told is a different situation with the same remedy, and the page keeps them apart.
|
||||
if !strings.Contains(rendered, "has never been sent anything") {
|
||||
t.Fatalf("the page does not separate never told from out of date:\n%s", rendered)
|
||||
}
|
||||
if !strings.Contains(rendered, "push --behind") {
|
||||
t.Fatalf("the page does not say what sends them:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
|
||||
// And says so when there is nothing waiting, rather than leaving the question unanswered.
|
||||
func TestAMeshWithNothingWaitingSaysSo(t *testing.T) {
|
||||
rendered := render(t, viewOf(answers{nodes: []inventory.Node{{Name: "anchor"}}}))
|
||||
if !strings.Contains(rendered, "Every machine is running what the mesh would send it") {
|
||||
t.Fatalf("the page leaves the question unanswered:\n%s", rendered)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,834 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// asking a build machine for a module, and what came back.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// buildCommand builds a module from its source and records what came out.
|
||||
//
|
||||
// **Run where there is a container runtime**, which is why it is a command rather than something
|
||||
// the control plane does on its own: building needs to run things on a machine, and what the
|
||||
// control plane may send a machine is bounded by the declaration language. This is the shape the
|
||||
// builder module will take when it is given work over the broker; today a person runs it, and the
|
||||
// mesh records the result the same way either way.
|
||||
// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the
|
||||
// rebuild a changed base needs, which nothing else asks for: their sources did not move, and
|
||||
// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too.
|
||||
func buildOn(ctx context.Context, base string, wait time.Duration) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
against, err := open.inventory.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var on []inventory.Entry
|
||||
for _, e := range held {
|
||||
if standsOnModule(e, base, against) {
|
||||
on = append(on, e)
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("nothing the mesh holds stands on %s\n", base)
|
||||
return nil
|
||||
}
|
||||
on = orderByBases(on, against)
|
||||
fmt.Printf("%d module(s) stand on %s:\n", len(on), base)
|
||||
var failed []string
|
||||
for _, e := range on {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", "))
|
||||
}
|
||||
fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base)
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("build", flag.ContinueOnError)
|
||||
ref := set.String("ref", "", "the branch, tag or commit to build")
|
||||
// A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's
|
||||
// root, which is the ordinary case and why this is a flag rather than a second argument.
|
||||
path := set.String("path", "", "the module's directory inside the repository")
|
||||
wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer")
|
||||
dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing")
|
||||
// Every module whose source has moved, rather than one named repository.
|
||||
//
|
||||
// **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh
|
||||
// already knows which modules are behind their source, so making a person read that list and
|
||||
// retype each repository is asking them to be the loop. Naming a repository and asking which
|
||||
// ones need building are different requests, so they are not combined.
|
||||
behind := set.Bool("behind", false, "every module the mesh holds older than its source has")
|
||||
on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs")
|
||||
// A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it
|
||||
// the repository is external, cloned exactly as given — see source.go.
|
||||
self := set.Bool("self", false, "the repository is a path on the forge holding the git seat")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *on != "" {
|
||||
if len(positionals) != 0 || *behind || *self {
|
||||
return errors.New("build --on <module> names a base and nothing else")
|
||||
}
|
||||
return buildOn(ctx, *on, *wait)
|
||||
}
|
||||
|
||||
if *behind {
|
||||
if len(positionals) != 0 || *self {
|
||||
return errors.New("build <repository> or build --behind, not both: one names a " +
|
||||
"repository and the other asks which need building")
|
||||
}
|
||||
return buildBehind(ctx, *wait)
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("build <repository> [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on <module>")
|
||||
}
|
||||
source := buildSource{Repository: positionals[0]}
|
||||
if *self {
|
||||
if err := onASeat(source.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
source.Seat = gitSeat
|
||||
}
|
||||
|
||||
if *dryRun {
|
||||
return buildAndShow(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
return buildOne(ctx, source, *path, *ref, *wait)
|
||||
}
|
||||
|
||||
// buildFrom turns what a builder said into what the mesh keeps.
|
||||
//
|
||||
// **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder
|
||||
// says `<registry>:<port>/<module>/<artifact>@sha256:…`; the mesh records the artifact-store
|
||||
// reference and composes the store's address back in where a reference is used. `against` — what
|
||||
// the build stood on, the catalogue's edge — is recorded the same way, so an edge names a module's
|
||||
// artifact and not the machine it was pulled from.
|
||||
func buildFrom(result link.BuildResult) inventory.Build {
|
||||
kept := inventory.Build{
|
||||
ID: result.ID, Repository: result.Repository, Ref: result.Ref,
|
||||
Commit: result.Commit, On: result.On, Failed: result.Failed,
|
||||
// **What the announcement carries, kept rather than discarded** (novox/hq 04-ISSUES/050).
|
||||
// The catalogue turns the manifest into requires/provides edges and `against` into build
|
||||
// edges, and it is not always listening when a build happens — on a fresh mesh it cannot
|
||||
// be, for exactly the modules it needs most. Keeping them is what makes a replay able to
|
||||
// rebuild the graph rather than a list of names.
|
||||
Path: result.Path,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
if asked, ok := link.BuildAskedAt(result.ID); ok {
|
||||
kept.Asked = asked
|
||||
}
|
||||
for _, ref := range result.Against {
|
||||
kept.Against = append(kept.Against, catalogue.Recorded(ref))
|
||||
}
|
||||
for _, r := range result.Read {
|
||||
kept.Read = append(kept.Read, inventory.ReadRepository{Repository: r.Repository, Ref: r.Ref})
|
||||
}
|
||||
var announced []inventory.Artifact
|
||||
for _, made := range result.Made {
|
||||
announced = append(announced, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
kept.Made = append(kept.Made, inventory.Artifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: catalogue.Recorded(made.Reference),
|
||||
})
|
||||
}
|
||||
kept.Manifest = recordedManifest(result.Manifest, announced)
|
||||
// The module name comes from the manifest, which only exists when the build got that far.
|
||||
if len(kept.Manifest) > 0 {
|
||||
if m, err := catalogue.ParseManifest(kept.Manifest); err == nil {
|
||||
kept.Module = m.Module
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// buildsCommand says what has been built lately.
|
||||
func buildsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("builds", flag.ContinueOnError)
|
||||
limit := set.Int("n", 20, "how many to show")
|
||||
logOf := set.String("log", "", "a build's id: print what the build machine said, line by line")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *logOf != "" {
|
||||
return buildLog(ctx, *logOf)
|
||||
}
|
||||
module := ""
|
||||
if len(positionals) == 1 {
|
||||
module = positionals[0]
|
||||
} else if len(positionals) > 1 {
|
||||
return errors.New("builds [<module>] [-n N]")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
builds, err := inv.Builds(ctx, module, *limit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(builds) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never look
|
||||
// the same, and getting here means the store answered.
|
||||
if module != "" {
|
||||
fmt.Printf("nothing has been built for %s\n", module)
|
||||
return nil
|
||||
}
|
||||
fmt.Println("nothing has been built yet")
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, b := range builds {
|
||||
what := b.Module
|
||||
if what == "" {
|
||||
// It failed before knowing what it was building, which is most of the interesting
|
||||
// failures. The repository is what a person has to go and look at.
|
||||
what = "?"
|
||||
}
|
||||
outcome := "built " + short(b.Commit)
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Printf(" %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Printf(" at %s", b.Ref)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, made := range b.Made {
|
||||
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
fmt.Printf(" %s\n", firstLine(b.Failed))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// builderCommand issues a build machine its own broker credential.
|
||||
//
|
||||
// **A build machine is not a node**, and giving it a node's account would let it read another
|
||||
// machine's declarations. This is narrower and different: read the build queue, write the
|
||||
// exchange and an asker's reply queue, and nothing else.
|
||||
//
|
||||
// Issued rather than assumed, because until this the builder used whatever credential it was
|
||||
// handed — which in practice meant the broker's own administrative one. A program documented as
|
||||
// holding its own credential and given somebody else's is worse than one with no story at all.
|
||||
func builderCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("builder issue", flag.ContinueOnError)
|
||||
// Which machine will use it. Given, the credential is delivered by the mesh rather than
|
||||
// printed for somebody to carry — which is the difference between the builder being a module
|
||||
// and being a program somebody configures.
|
||||
forNode := set.String("node", "",
|
||||
"the machine that will run it, so the mesh delivers the credential instead of printing it")
|
||||
module := set.String("module", "builder", "the module on that machine that will read it")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 2 || positionals[0] != "issue" {
|
||||
return errors.New("builder issue <name> [--node <machine>]")
|
||||
}
|
||||
name := positionals[1]
|
||||
|
||||
// **The build machine's credential is a module's credential** (novox/hq ADR 0131, design 28
|
||||
// task 5.5): minted into the mesh's records and sealed to the machine as the builder module's
|
||||
// broker secret, usable at the next push — the same act `module issue` performs, and the same
|
||||
// account the composed user list carries. Nothing is created on a server; the bus reads the list.
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, known := shelf[*module]
|
||||
if !known {
|
||||
return fmt.Errorf("%s is not in the catalogue; `module add` it first", *module)
|
||||
}
|
||||
fmt.Printf("build machine %s: ", name)
|
||||
node := *forNode
|
||||
if node == "" {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == *module && len(e.On) > 0 {
|
||||
node = e.On[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if node == "" {
|
||||
return fmt.Errorf("%s is assigned nowhere; `assign <machine> %s` first, or say --node", *module, *module)
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueOnTheNewBus(ctx, inv, m, node, address)
|
||||
}
|
||||
|
||||
// buildBehind builds every module the mesh holds older than its source has.
|
||||
//
|
||||
// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already
|
||||
// records where each module came from and what its source last had; until this, a person read
|
||||
// that list and retyped each repository — which is a person being the loop, and the thing a
|
||||
// pipeline was doing before it was taken away.
|
||||
//
|
||||
// Each is built and recorded on its own. **One failing does not stop the others**, for the same
|
||||
// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad
|
||||
// repository holds back nine good ones is a mesh where nobody dares add the tenth.
|
||||
func buildBehind(ctx context.Context, wait time.Duration) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
held, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stale []inventory.Entry
|
||||
for _, e := range held {
|
||||
if !e.Source.Current() {
|
||||
stale = append(stale, e)
|
||||
}
|
||||
}
|
||||
if len(stale) == 0 {
|
||||
// Said rather than doing nothing quietly: "nothing needed building" and "this did not
|
||||
// run" must never look the same.
|
||||
fmt.Println("every module the mesh holds is what its source last had")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("%d module(s) behind their source:\n", len(stale))
|
||||
for _, e := range stale {
|
||||
fmt.Printf(" %s %s < %s\n",
|
||||
e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head))
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// Bases first: a module built before the module it stands on is built against the old one
|
||||
// and reports success (novox/hq 04-ISSUES/131).
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stale = orderByBases(stale, against)
|
||||
|
||||
var failed []string
|
||||
for _, e := range stale {
|
||||
fmt.Printf("--- %s\n", e.Manifest.Module)
|
||||
// Its own recorded ref, not its head commit: a module tracking a branch should be built
|
||||
// from that branch, and pinning to the commit the mesh happened to notice would quietly
|
||||
// turn a tracked branch into a pin.
|
||||
source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil {
|
||||
fmt.Printf(" %v\n", err)
|
||||
failed = append(failed, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
|
||||
if len(failed) > 0 {
|
||||
return fmt.Errorf("%d of %d could not be built: %s",
|
||||
len(failed), len(stale), strings.Join(failed, ", "))
|
||||
}
|
||||
fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n",
|
||||
len(stale))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildOne asks a build machine for one repository and records everything that came back.
|
||||
//
|
||||
// Separated from the command so `--behind` can walk a list without a second path to the same act.
|
||||
func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
_, err := buildOneAsked(ctx, source, path, ref, wait, false)
|
||||
return err
|
||||
}
|
||||
|
||||
// buildOneAsked is buildOne answering the id it asked with — what a plan keeps to match the outcome
|
||||
// by (novox/hq ADR 0219) — and, for an ask not waited for, optionally a dry run: built and looked
|
||||
// at, never taken in (issue 240), which is what `replay` asks unless told to register.
|
||||
func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wait time.Duration,
|
||||
dryRun bool) (string, error) {
|
||||
if dryRun && wait != 0 {
|
||||
return "", errors.New("a dry run waited for is `build --dry-run`")
|
||||
}
|
||||
// Before anything is asked of a builder: a source on a seat nobody holds is refused here, with
|
||||
// the reason, rather than sent to a machine to fail at `git clone`.
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// Correlated by something the control plane makes, not by the module's name: two builds of one
|
||||
// module can be in flight, and the second answer is not the first one's.
|
||||
request := link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository,
|
||||
Path: path,
|
||||
Ref: ref,
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
DryRun: dryRun,
|
||||
}
|
||||
fmt.Printf("asked for %s", source)
|
||||
if source.Seat != "" {
|
||||
fmt.Printf(" (%s)", repository)
|
||||
}
|
||||
// The id is how a person follows this build while it runs: `builds --log <id>`.
|
||||
fmt.Printf(" as %s", request.ID)
|
||||
if path != "" {
|
||||
fmt.Printf(" at %s", path)
|
||||
}
|
||||
if ref != "" {
|
||||
fmt.Printf(" on %s", ref)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
seat := buildSeatHeld(ctx)
|
||||
ask, err := askOverOn(seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer ask.Close()
|
||||
fmt.Printf(" of %s\n", seat)
|
||||
|
||||
if wait == 0 {
|
||||
// Asked and not waited for (novox/hq issue 176): the outcome is the role's event, and the
|
||||
// controller takes it in — records the build, registers the module — whether or not anybody
|
||||
// is still here. A tool call cannot hold a connection for the minutes a build takes; it
|
||||
// follows the build by its id instead.
|
||||
if err := ask.Ask(ctx, request); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if dryRun {
|
||||
fmt.Printf("asked as a dry run, not waited for: `builds --log %s` follows it as it runs; "+
|
||||
"its outcome is not taken in\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
fmt.Printf("asked, not waited for: `builds --log %s` follows it as it runs, and `builds` "+
|
||||
"shows what came of it; the module is registered when the outcome comes\n", request.ID)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
result, err := ask.Submit(ctx, request, wait)
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
defer open.Close()
|
||||
manifest, kept, err := takeIn(ctx, open.inventory, result)
|
||||
if err != nil {
|
||||
return request.ID, err
|
||||
}
|
||||
// Said as recorded: what each artifact is, not where this builder happened to push it.
|
||||
for _, made := range kept.Made {
|
||||
fmt.Printf(" %-12s %s %s\n", made.Name, made.Kind, made.Reference)
|
||||
}
|
||||
fmt.Printf("\n%s %s, built on %s from %s\n",
|
||||
manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, open.inventory, manifest.Module)
|
||||
fmt.Printf(" run `assign <node> %s` to put it somewhere\n", manifest.Module)
|
||||
return request.ID, nil
|
||||
}
|
||||
|
||||
// saysWhenThePolicyActs tells whoever built a module that its upgrade policy will send the
|
||||
// result on at once (novox/hq issue 126, ADR 0163): a person choreographing a data move must
|
||||
// know which module will not wait for them.
|
||||
func saysWhenThePolicyActs(ctx context.Context, inv *inventory.Inventory, module string) {
|
||||
if u, err := inv.UpgradeOf(ctx, module); err == nil && u.RollOut {
|
||||
how := "one machine at a time"
|
||||
if u.Together {
|
||||
how = "every machine at once"
|
||||
}
|
||||
fmt.Printf(" %s rolls out on build: the machines running it are sent this now, %s — "+
|
||||
"`upgrade %s record` first if something must move before it does\n", module, how, module)
|
||||
}
|
||||
}
|
||||
|
||||
// takeIn is what the mesh does with a build's outcome, whoever hears it: the waiting command and
|
||||
// the daemon that follows the role's events both come here (novox/hq issue 176), so a build's
|
||||
// result reaches the catalogue whether or not the asker was still listening.
|
||||
//
|
||||
// Kept before it is judged. A failed build that leaves no trace is indistinguishable from one
|
||||
// nobody asked for, and the difference is the whole of whether somebody should be looking at
|
||||
// something. Then parsed with the same parser a hand-written manifest goes through — a second path
|
||||
// would be a second thing to disagree about what a manifest is — and registered with where it came
|
||||
// from: **for a source on a seat, as the path and the seat, never the URL just cloned** (ADR 0111),
|
||||
// which the request carried and the outcome echoes. A definition naming an installation is refused
|
||||
// here, where it would enter the catalogue; the build stays recorded and the refusal says which.
|
||||
//
|
||||
// Idempotent: the same outcome taken in twice registers the same module twice, which is one row
|
||||
// written with the same values.
|
||||
func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (
|
||||
catalogue.Manifest, inventory.Build, error) {
|
||||
kept := buildFrom(result)
|
||||
if err := inv.RecordBuild(ctx, kept); err != nil {
|
||||
return catalogue.Manifest{}, kept, err
|
||||
}
|
||||
if result.Failed != "" {
|
||||
// The builder's own words. Wrapping them in something about the control plane would put
|
||||
// two explanations between a person and a build log.
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s could not build %s:\n%s",
|
||||
result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, kept, fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
}
|
||||
recorded := inventory.Source{
|
||||
Repository: result.Repository, Path: result.Path, Ref: result.Ref,
|
||||
BuiltFrom: result.Commit, Head: result.Commit,
|
||||
// What it stood on, so registration can judge a built manifest's base (to-be 38 WP2.4).
|
||||
Against: kept.Against,
|
||||
// When it was asked, so an older request heard later does not replace a newer one
|
||||
// (novox/hq 04-ISSUES/219).
|
||||
Asked: kept.Asked,
|
||||
}
|
||||
if result.Source != nil && result.Source.Seat != "" {
|
||||
recorded.Repository, recorded.Seat = result.Source.Repository, result.Source.Seat
|
||||
}
|
||||
// **A build at a commit does not change the branch a module follows** (novox/hq 04-ISSUES/215):
|
||||
// the commit is built and recorded as what it was built from, and the module keeps following
|
||||
// what it followed before — the repository's default branch for one new to the catalogue.
|
||||
if followedBranch(result.Ref) == "" && result.Ref != "" {
|
||||
recorded.Ref = ""
|
||||
if was, err := inv.SourceOf(ctx, manifest.Module); err == nil {
|
||||
recorded.Ref = followedBranch(was.Ref)
|
||||
}
|
||||
}
|
||||
if err := namesNoInstallation(manifest); err != nil {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w",
|
||||
result.On, result.Repository, short(result.Commit), err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifest, recorded); err != nil {
|
||||
if errors.Is(err, inventory.ErrSuperseded) {
|
||||
return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w",
|
||||
result.On, manifest.Module, short(result.Commit), err)
|
||||
}
|
||||
return manifest, kept, err
|
||||
}
|
||||
// The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather
|
||||
// than on a timer of its own: this is the only moment either is true. Never fatal — the build
|
||||
// worked and the module is registered.
|
||||
collect(ctx, inv)
|
||||
return manifest, kept, nil
|
||||
}
|
||||
|
||||
// buildAndShow builds and prints the manifest without recording anything.
|
||||
func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error {
|
||||
repository, err := cloneFrom(ctx, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
ask, err := askOverOn(buildSeatHeld(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ask.Close()
|
||||
|
||||
result, err := ask.Submit(ctx, link.BuildRequest{
|
||||
ID: link.NewBuildID(time.Now()),
|
||||
Repository: repository, Path: path, Ref: ref,
|
||||
Held: heldBy(ctx), Seats: seatBases(ctx),
|
||||
DryRun: true,
|
||||
}, wait)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.Failed != "" {
|
||||
return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed)
|
||||
}
|
||||
manifest, err := catalogue.ParseManifest(result.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s built %s and what came back is not a manifest: %w",
|
||||
result.On, result.Repository, err)
|
||||
}
|
||||
body, err := json.MarshalIndent(manifest, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// answers is what the three questions came back with, read once.
|
||||
type answers struct {
|
||||
wrong []inventory.Doing
|
||||
nodes []inventory.Node
|
||||
quiet []inventory.Node
|
||||
behind map[string][]string
|
||||
sources map[string]inventory.Source
|
||||
// waiting is every machine not running what the mesh would send it.
|
||||
waiting []inventory.Machine
|
||||
// reported is every machine's last word beside when it was last sent a declaration — the
|
||||
// pair that answers "has it caught up", which waiting alone cannot (the sent digest is
|
||||
// recorded at send, not at apply).
|
||||
reported []inventory.Reported
|
||||
// plans is what the last merges produced and where each stands (novox/hq ADR 0162).
|
||||
plans []inventory.Plan
|
||||
// paused is whether the build seat takes work, which a plan waiting on it says (ADR 0219).
|
||||
paused pauseView
|
||||
// refused is why a machine cannot be worked out at all, by name. A different thing from every
|
||||
// other answer here: those are about a machine that was told something, and this is about one
|
||||
// that cannot be told anything — it never reaches waiting, because nothing was computed for it
|
||||
// to compare against, so without this a wholly blocked mesh reads as a well one.
|
||||
refused map[string]string
|
||||
// network is why the private network could not be computed, when it could not. Almost always
|
||||
// a consequence of the refusals above: a node that does not resolve is not on the network, and
|
||||
// a mesh whose hub is that node has no hub.
|
||||
network string
|
||||
// filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168): what filters it beyond the mesh's own, the runtime's plumbing and bans, by name — a
|
||||
// predecessor's chain, a found firewall in force again. Such a machine is not "all well".
|
||||
filtered map[string]inventory.Filtering
|
||||
// untaken is, per machine, each assigned module whose resources the machine is holding as it
|
||||
// found them, and how many — a module that was assigned, sent, and is running none of what it
|
||||
// declares because nothing has taken it (novox/hq ADR 0100, 04-ISSUES/125).
|
||||
//
|
||||
// **Its absence cost an outage.** The module was assigned, the push reported success, this
|
||||
// command said the machine was doing everything it was told, and the module's three containers
|
||||
// did not exist. On the strength of those reports the predecessor's proxy was stopped and every
|
||||
// public name on the machine went dark. The holds were correct; they were recorded only in the
|
||||
// machine's own state file, and the one visible symptom was a count that did not add up.
|
||||
untaken map[string]map[string]int
|
||||
// unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds (novox/hq ADR 0207), with the modules that could hold it. Reported, not
|
||||
// refused, until the switch — and while there is any, the mesh is not all well: the order the
|
||||
// machines' modules are built in is the mesh's to keep, and this is where it says it is not kept.
|
||||
unheld []catalogue.Unheld
|
||||
// failing is every consumer a provider says it keeps failing (novox/hq ADR 0224): a provider's
|
||||
// journal was the only place that said so for a day (04-ISSUES/179).
|
||||
failing []inventory.ProviderStanding
|
||||
}
|
||||
|
||||
// heldBy is every artifact this mesh has built, for a build that may need one as its base.
|
||||
//
|
||||
// **A failure here is not a failure to build.** A module that names no base does not need this at
|
||||
// all, and one that does gets a refusal naming exactly what is missing — which is a better sentence
|
||||
// than a build command refusing to start because a query did not run. So the store not opening is
|
||||
// reported and the build goes ahead without it.
|
||||
//
|
||||
// Routed through the artifact store as the network reaches it now (novox/hq 04-ISSUES/102): a
|
||||
// base is recorded by digest and path, and a build machine needs something it can pull.
|
||||
func heldBy(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built, so a module naming a "+
|
||||
"base will be told that base is missing: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.inventory.Held(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what this mesh has built: %v\n", err)
|
||||
return nil
|
||||
}
|
||||
address, err := whereABuilderReachesTheStore(ctx, open.inventory)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store on this mesh's network, so a "+
|
||||
"module naming a base will be handed a reference nothing can fetch: %v\n", err)
|
||||
return held
|
||||
}
|
||||
if address == "" {
|
||||
return held
|
||||
}
|
||||
routed := make(map[string]string, len(held))
|
||||
for repository, reference := range held {
|
||||
routed[repository] = catalogue.Rerouted(reference, address)
|
||||
}
|
||||
return routed
|
||||
}
|
||||
|
||||
// askOver opens the way a build is asked for, on whichever bus the mesh is on.
|
||||
//
|
||||
// **One place chooses**, as everywhere else the bus change went (novox/hq ADR 0116 step 5). On the bus
|
||||
// the mesh runs on today this needs the controller's own connection, so it is handed one; on the bus
|
||||
// being built it dials, because a build request is a one-shot and holds nothing else.
|
||||
func askOverOn(seat string) (link.Builders, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return link.BuildsOverNATSOn(address, seat)
|
||||
}
|
||||
|
||||
// buildSeatHeld is the build role to ask: the one some assigned module claims (novox/hq ADR 0190,
|
||||
// the handover). Read from the catalogue at ask time, because the answer changes exactly once, the
|
||||
// moment the first build-agent is assigned — and a controller that asked the new role before then
|
||||
// would queue work nothing takes, while the outcome that registers build-agent itself has to come
|
||||
// from the old builder. When the catalogue cannot be read the current role is asked, said aloud.
|
||||
func buildSeatHeld(ctx context.Context) string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read what is assigned, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue, so the build is asked of %s: %v\n",
|
||||
link.TheBuildMachine, err)
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
return buildSeatAmong(entries)
|
||||
}
|
||||
|
||||
// buildSeatAmong is the rule, over what the catalogue holds: the current build role when any
|
||||
// assigned module claims it; else the retired role while an assigned module still claims that; else
|
||||
// the current role, which is where every ask goes once the handover is done.
|
||||
func buildSeatAmong(entries []inventory.Entry) string {
|
||||
heldBefore := false
|
||||
for _, e := range entries {
|
||||
if len(e.On) == 0 {
|
||||
continue
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachine) {
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
if e.Manifest.ClaimsSeat(link.TheBuildMachineBefore) {
|
||||
heldBefore = true
|
||||
}
|
||||
}
|
||||
if heldBefore {
|
||||
return link.TheBuildMachineBefore
|
||||
}
|
||||
return link.TheBuildMachine
|
||||
}
|
||||
|
||||
// buildLog prints everything a build machine said about one build, read back from the bus.
|
||||
//
|
||||
// **From the stream, not from a record** (novox/hq ADR 0157). A build's lines are the role's own
|
||||
// events under the build's id, retained with every other event; the mesh keeps no second copy. Read
|
||||
// with a consumer of its own that is gone when this returns, so nothing accumulates in the server
|
||||
// for the reading, and filtered by subject, so one build's lines are all that travel.
|
||||
func buildLog(ctx context.Context, id string) error {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus to read a build's log: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
|
||||
// Under whichever build role did it: a build asked of the retired role during the handover
|
||||
// (ADR 0190) said its lines as that role's events, and a reader should not have to know which.
|
||||
lines := link.BuildLogOf("*", id)
|
||||
sub, err := js.Context().PullSubscribe(lines, "",
|
||||
nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone())
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot read %s from the bus: %w", lines, err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
|
||||
printed := 0
|
||||
for {
|
||||
batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second))
|
||||
if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) {
|
||||
return fmt.Errorf("reading a build's log: %w", err)
|
||||
}
|
||||
for _, msg := range batch {
|
||||
var line link.BuildLine
|
||||
if err := json.Unmarshal(msg.Data, &line); err != nil {
|
||||
fmt.Printf(" ? %s\n", string(msg.Data))
|
||||
continue
|
||||
}
|
||||
at := line.At
|
||||
if t, err := time.Parse(time.RFC3339Nano, line.At); err == nil {
|
||||
at = t.Local().Format("15:04:05")
|
||||
}
|
||||
fmt.Printf("%s %4d [%s] %s\n", at, line.Seq, line.Step, line.Message)
|
||||
printed++
|
||||
}
|
||||
if len(batch) < 200 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if printed == 0 {
|
||||
fmt.Printf("nothing on the bus for build %s: no build by that id in the last week, or a build "+
|
||||
"machine older than this that said nothing while building\n", id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func claiming(module, seat string, on ...string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: seat}}},
|
||||
On: on,
|
||||
}
|
||||
}
|
||||
|
||||
// The controller asks the build role that has a holder (novox/hq ADR 0190 handover): the retired
|
||||
// one while only the builder is assigned, the current one from the first build-agent on, and the
|
||||
// current one when nothing holds either — where every ask goes once the handover is done.
|
||||
func TestTheControllerAsksTheBuildRoleThatHasAHolder(t *testing.T) {
|
||||
onlyTheBuilder := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine), // registered, assigned nowhere yet
|
||||
}
|
||||
if got := buildSeatAmong(onlyTheBuilder); got != link.TheBuildMachineBefore {
|
||||
t.Errorf("with only the builder assigned, asked %q", got)
|
||||
}
|
||||
bothHeld := []inventory.Entry{
|
||||
claiming("builder", link.TheBuildMachineBefore, "anchor"),
|
||||
claiming("build-agent", link.TheBuildMachine, "home-server"),
|
||||
}
|
||||
if got := buildSeatAmong(bothHeld); got != link.TheBuildMachine {
|
||||
t.Errorf("with a build-agent assigned anywhere, asked %q", got)
|
||||
}
|
||||
neither := []inventory.Entry{claiming("builder", link.TheBuildMachineBefore)}
|
||||
if got := buildSeatAmong(neither); got != link.TheBuildMachine {
|
||||
t.Errorf("with no holder of either, asked %q, want the current role", got)
|
||||
}
|
||||
if got := buildSeatAmong(nil); got != link.TheBuildMachine {
|
||||
t.Errorf("an empty catalogue asks %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A build's outcome is taken in the same way whoever hears it (novox/hq issue 176): recorded, and
|
||||
// the module registered with its source as the seat and path when the request said so — never the
|
||||
// URL. A definition naming an installation is recorded and not registered; a failure is recorded
|
||||
// and said.
|
||||
func TestABuildHeardIsRecordedAndRegistered(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"})
|
||||
m, _, err := takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop",
|
||||
Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Module != "shop" {
|
||||
t.Fatalf("registered %q", m.Module)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, held := shelf["shop"]; !held {
|
||||
t.Fatal("the module a heard build produced is not in the catalogue")
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "shop")
|
||||
if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" {
|
||||
t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err)
|
||||
}
|
||||
builds, err := open.inventory.Builds(ctx, "shop", 5)
|
||||
if err != nil || len(builds) != 1 || builds[0].ID != "b-1" {
|
||||
t.Fatalf("the build is not recorded once: %v %v", builds, err)
|
||||
}
|
||||
|
||||
named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{
|
||||
map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}})
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{
|
||||
ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named})
|
||||
if err == nil || !strings.Contains(err.Error(), "does not register it") {
|
||||
t.Fatalf("a definition naming an installation was taken in: %v", err)
|
||||
}
|
||||
if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" {
|
||||
t.Fatal("the refused module was registered anyway")
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 {
|
||||
t.Fatalf("the refused build was not recorded: %v", builds)
|
||||
}
|
||||
|
||||
_, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"})
|
||||
if err == nil || !strings.Contains(err.Error(), "no compiler") {
|
||||
t.Fatalf("a failure is said in the builder's words: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the
|
||||
// module keeps following the branch it followed — a new one, the default branch.
|
||||
func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"})
|
||||
result := func(id, ref, commit string) link.BuildResult {
|
||||
return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-1", "main", "1111111aaaa")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
src, err := open.inventory.SourceOf(ctx, "unifi")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" {
|
||||
t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom)
|
||||
}
|
||||
|
||||
// One new to the catalogue, first built at a commit, follows the default branch.
|
||||
other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"})
|
||||
r := result("b-3", "deadbeef", "deadbeefcafe")
|
||||
r.Manifest, r.Path = other, "modules/letta"
|
||||
if _, _, err := takeIn(ctx, open.inventory, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" {
|
||||
t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered,
|
||||
// so a push sends what the newer request built.
|
||||
func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
result := func(asked time.Time, image string) link.BuildResult {
|
||||
manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image})
|
||||
return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git",
|
||||
Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
}
|
||||
if _, _, err := takeIn(ctx, open.inventory, result(newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9"))
|
||||
if !errors.Is(err, inventory.ErrSuperseded) {
|
||||
t.Fatalf("the older request's outcome was taken in as current: %v", err)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "4bcd5f73" {
|
||||
t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got)
|
||||
}
|
||||
if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 {
|
||||
t.Errorf("the late build was not recorded: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC)
|
||||
if got, ok := link.BuildAskedAt(link.NewBuildID(at)); !ok || !got.Equal(at) {
|
||||
t.Errorf("read back %v %v; want %v", got, ok, at)
|
||||
}
|
||||
if got, ok := link.BuildAskedAt("build-1791064317392539762"); !ok || got.Format(time.TimeOnly) != "21:51:57" {
|
||||
t.Errorf("the incident's id reads as %v %v", got, ok)
|
||||
}
|
||||
for _, id := range []string{"b-1", "build-2", "build-", "build-x", ""} {
|
||||
if _, ok := link.BuildAskedAt(id); ok {
|
||||
t.Errorf("%q read as a request time", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// The bus's own certificate, made by the mesh rather than borrowed from an image.
|
||||
//
|
||||
// **The foundation asked a third-party image for a tool it never said must be there** (novox/hq
|
||||
// 04-ISSUES/146). The bootstrap made this certificate by running `openssl` inside the broker's
|
||||
// image, which worked while the broker was one that happened to carry it and stopped the day the
|
||||
// bus changed: the new one has a shell and no openssl, so the step exited 127 and no mesh could be
|
||||
// raised. Substituting another image the bundle names does not help — none of them carry it
|
||||
// either.
|
||||
//
|
||||
// So the program that needs a certificate makes one. It is the mesh's own binary, already on the
|
||||
// machine at this point in the bootstrap (the schema step ran it), and it needs nothing from the
|
||||
// image it writes into but a mounted directory.
|
||||
//
|
||||
// **Self-signed, and that is the design** — a host pins this server's exact certificate and
|
||||
// authenticates with a password (novox/hq ADR 0004). There is no authority above it to ask, and at
|
||||
// this moment in a bootstrap there is no mesh to ask one of.
|
||||
//
|
||||
// Idempotent, because the step is applied again on every reconcile and a second certificate would
|
||||
// be one the hosts that pinned the first no longer believe.
|
||||
|
||||
// busCertificateNames is what the bus is reached by: the container name on a mesh network, and the
|
||||
// loopback address the machine's own foundation dials.
|
||||
var busCertificateNames = []string{"mesh-broker"}
|
||||
|
||||
const busCertificateLife = 10 * 365 * 24 * time.Hour
|
||||
|
||||
// busCertificate makes the bus's certificate in a directory, or says whether one is there.
|
||||
//
|
||||
// broker certificate --into /tls make it if it is not there
|
||||
// broker certificate --check --into /tls exit non-zero unless a usable pair is
|
||||
func busCertificate(args []string) error {
|
||||
into, check := "", false
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--check":
|
||||
check = true
|
||||
case "--into":
|
||||
if i+1 >= len(args) {
|
||||
return errors.New("--into needs a directory")
|
||||
}
|
||||
into = args[i+1]
|
||||
i++
|
||||
default:
|
||||
return fmt.Errorf("broker certificate [--check] --into <directory>: %q", args[i])
|
||||
}
|
||||
}
|
||||
if into == "" {
|
||||
return errors.New("broker certificate [--check] --into <directory>")
|
||||
}
|
||||
crt, key := filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")
|
||||
|
||||
if usable, err := busCertificateUsable(crt, key); err != nil {
|
||||
return err
|
||||
} else if usable {
|
||||
fmt.Printf("the bus already has a certificate at %s, and it was left alone\n", crt)
|
||||
return nil
|
||||
}
|
||||
if check {
|
||||
// Said as a failure, because that is what the caller asked: a bootstrap's verify runs
|
||||
// this and a false answer is what makes the step run.
|
||||
return fmt.Errorf("no usable certificate and key at %s", into)
|
||||
}
|
||||
return writeBusCertificate(crt, key)
|
||||
}
|
||||
|
||||
// busCertificateUsable says whether a certificate and its key are both there and parse.
|
||||
//
|
||||
// Both, and parsed rather than stat'ed: a half-written pair is the state a bootstrap interrupted
|
||||
// between the two files leaves behind, and a step that treated it as done would hand the server a
|
||||
// certificate with no key and report success.
|
||||
func busCertificateUsable(crt, key string) (bool, error) {
|
||||
certPEM, err := os.ReadFile(crt)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
keyPEM, err := os.ReadFile(key)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err := tlsPairParses(certPEM, keyPEM); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func tlsPairParses(certPEM, keyPEM []byte) (*x509.Certificate, error) {
|
||||
block, _ := pem.Decode(certPEM)
|
||||
if block == nil || block.Type != "CERTIFICATE" {
|
||||
return nil, errors.New("not a certificate")
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyBlock, _ := pem.Decode(keyPEM)
|
||||
if keyBlock == nil {
|
||||
return nil, errors.New("not a key")
|
||||
}
|
||||
if _, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes); err != nil {
|
||||
if _, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return certificate, nil
|
||||
}
|
||||
|
||||
func writeBusCertificate(crt, key string) error {
|
||||
private, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: serial,
|
||||
Subject: pkix.Name{CommonName: busCertificateNames[0]},
|
||||
DNSNames: busCertificateNames,
|
||||
IPAddresses: []net.IP{net.ParseIP("127.0.0.1")},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(busCertificateLife),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, template, template, &private.PublicKey, private)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pkcs8, err := x509.MarshalPKCS8PrivateKey(private)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// **The key first, and only then the certificate**, so the pair a reader finds is never a
|
||||
// certificate whose key has not been written yet — the one order in which an interruption
|
||||
// leaves something that looks finished (novox/hq 04-ISSUES/014, a key present and unusable).
|
||||
if err := os.WriteFile(key, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8}), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(crt, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("made the bus a certificate for %v, valid until %s\n %s\n %s\n",
|
||||
busCertificateNames, template.NotAfter.Format(time.RFC3339), crt, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
// busAccounts writes the mesh's composed user list to a file.
|
||||
//
|
||||
// **For genesis, where no declaration can deliver it** (novox/hq 04-ISSUES/146). Everywhere else
|
||||
// the list reaches the machine running the bus as a resource of the module that holds it — which
|
||||
// requires that machine to be an enrolled node, and at genesis it is not: the first node cannot
|
||||
// enrol because the account it would enrol with cannot be composed onto a bus it has no declaration
|
||||
// for. The installer breaks that circle by placing the file itself, once, and the module takes the
|
||||
// file over from its first push.
|
||||
//
|
||||
// The same composition, not a second one: this asks the store for the same records and renders them
|
||||
// with the same composer the declaration uses. A genesis that hand-wrote an account would be a
|
||||
// second statement of who may say what, able to disagree with the first.
|
||||
//
|
||||
// **It writes to standard output unless told a file**, and that is the point: the control plane
|
||||
// composes and says what it composed, and whoever is raising the machine puts it where that
|
||||
// machine's bus reads it. A control plane that wrote into the bus's own directory would have to
|
||||
// know where that is and how to make the server re-read it — which is the module's knowledge, and
|
||||
// the module is what takes this over on the first push.
|
||||
//
|
||||
// broker accounts > /var/lib/mesh-bus-conf/accounts.conf
|
||||
func busAccounts(ctx context.Context, args []string) error {
|
||||
into := ""
|
||||
for i := 0; i < len(args); i++ {
|
||||
switch args[i] {
|
||||
case "--into":
|
||||
if i+1 >= len(args) {
|
||||
return errors.New("--into needs a file")
|
||||
}
|
||||
into = args[i+1]
|
||||
i++
|
||||
default:
|
||||
return fmt.Errorf("broker accounts --into <file>: %q", args[i])
|
||||
}
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := open.inventory.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
filled, missing := broker.WithPasswords(users, hashes)
|
||||
if len(missing) > 0 {
|
||||
// To standard error, always: the composed file may be going to standard output, and a
|
||||
// remark in the middle of it is a configuration the server refuses to parse.
|
||||
fmt.Fprintf(os.Stderr, "leaving out %d user(s) the mesh has minted no credential for: %s\n",
|
||||
len(missing), strings.Join(missing, ", "))
|
||||
}
|
||||
if len(filled) == 0 {
|
||||
return errors.New("not one user has a credential, so this list would refuse every " +
|
||||
"connection in the mesh")
|
||||
}
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if into == "" {
|
||||
fmt.Print(accounts)
|
||||
return nil
|
||||
}
|
||||
if err := os.WriteFile(into, []byte(accounts), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("wrote %d user(s) to %s\n", len(filled), into)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// novox/hq 04-ISSUES/146. The bootstrap could not make the bus a certificate: it asked an image for
|
||||
// `openssl` and the image it asks has none. What replaces it is this command, so what is checked is
|
||||
// what the bootstrap needs from it — a pair a TLS server can actually load, made once and only once.
|
||||
|
||||
func TestTheBusCertificateLoadsAsAServersWould(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatalf("the bus could not be given a certificate: %v", err)
|
||||
}
|
||||
|
||||
// The check a cheaper test would not make. The key was present and valid and the server could
|
||||
// not start, once, because nothing loaded the pair the way a server loads it
|
||||
// (novox/hq 04-ISSUES/014).
|
||||
pair, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatalf("a TLS server cannot load what was written: %v", err)
|
||||
}
|
||||
leaf := pair.Leaf
|
||||
if leaf == nil {
|
||||
if leaf, err = x509.ParseCertificate(pair.Certificate[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := leaf.VerifyHostname("mesh-broker"); err != nil {
|
||||
t.Errorf("the certificate is not for the name the bus is reached by: %v", err)
|
||||
}
|
||||
if len(leaf.IPAddresses) == 0 || leaf.IPAddresses[0].String() != "127.0.0.1" {
|
||||
t.Errorf("the certificate does not cover the loopback address the foundation dials: %v", leaf.IPAddresses)
|
||||
}
|
||||
|
||||
// The key is not readable by anything else on the machine; the certificate is public and is.
|
||||
key, err := os.Stat(filepath.Join(into, "tls.key"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if key.Mode().Perm() != 0o600 {
|
||||
t.Errorf("the key is %v", key.Mode().Perm())
|
||||
}
|
||||
crt, err := os.Stat(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if crt.Mode().Perm() != 0o644 {
|
||||
t.Errorf("the certificate is %v, which the server runs as another user cannot read", crt.Mode().Perm())
|
||||
}
|
||||
}
|
||||
|
||||
// **Made once.** The step is applied again on every reconcile, and a second certificate is one the
|
||||
// hosts that pinned the first no longer believe (novox/hq ADR 0004).
|
||||
func TestTheBusCertificateIsMadeOnce(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := os.ReadFile(filepath.Join(into, "tls.crt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(first) != string(again) {
|
||||
t.Fatal("running it twice replaced the certificate every host had pinned")
|
||||
}
|
||||
}
|
||||
|
||||
// The verify half: false before, true after, which is what makes the bootstrap run the step at all.
|
||||
func TestTheCheckIsFalseUntilThereIsAPair(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||
t.Fatal("an empty directory reported a usable certificate")
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err != nil {
|
||||
t.Fatalf("the certificate it just made does not satisfy its own check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A half-written pair is not a pair. An interrupted bootstrap leaves exactly this, and a step that
|
||||
// called it done would hand the server a certificate with no key and report success.
|
||||
func TestACertificateWithoutItsKeyIsNotUsable(t *testing.T) {
|
||||
into := t.TempDir()
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(into, "tls.key")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := busCertificate([]string{"--check", "--into", into}); err == nil {
|
||||
t.Fatal("a certificate with no key passed the check")
|
||||
}
|
||||
if err := busCertificate([]string{"--into", into}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tls.LoadX509KeyPair(filepath.Join(into, "tls.crt"), filepath.Join(into, "tls.key")); err != nil {
|
||||
t.Fatalf("it did not replace the unusable pair: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhereToWriteIsRequired(t *testing.T) {
|
||||
if err := busCertificate(nil); err == nil || !strings.Contains(err.Error(), "--into") {
|
||||
t.Fatalf("it did not ask where to write: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// consoleWaits is how long the console waits for an answer (mesh-tools node-tools/internal/bus
|
||||
// RequestTimeout) — the shortest wait of a caller the mesh ships.
|
||||
const consoleWaits = 30 * time.Second
|
||||
|
||||
// **A call's one answer is never later than its caller or the bus allow** (novox/hq issue 265): a
|
||||
// holder answers within AnswerWithin, which must be inside both the console's wait and the window the
|
||||
// bus gives an answer. Before, the console waited 30s, the bus 60s, and a push ran as long as it ran.
|
||||
func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) {
|
||||
if link.AnswerWithin >= consoleWaits/2 {
|
||||
t.Errorf("a call answers within %s: not well inside the console's %s", link.AnswerWithin, consoleWaits)
|
||||
}
|
||||
if link.AnswerWithin >= broker.ResponseTTL {
|
||||
t.Errorf("a call answers within %s, after the bus stops permitting an answer at %s", link.AnswerWithin, broker.ResponseTTL)
|
||||
}
|
||||
}
|
||||
|
||||
// A push — named or through command — answers before it runs: it sends the machine holding the bus
|
||||
// first, and the broker reloading its user list forgets the answer it was about to permit.
|
||||
func TestAPushAnswersBeforeItSends(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want bool
|
||||
}{
|
||||
{"push", map[string]any{"node": "anchor"}, true},
|
||||
{"push", map[string]any{}, true},
|
||||
{"command", map[string]any{"command": "push anchor"}, true},
|
||||
{"command", map[string]any{"command": "push --behind"}, true},
|
||||
{"command", map[string]any{"command": "builds"}, false},
|
||||
{"status", map[string]any{}, false},
|
||||
{"assign", map[string]any{"node": "anchor", "module": "m"}, false},
|
||||
} {
|
||||
argv, err := argvFor(c.verb, c.args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %v: %v", c.verb, c.args, err)
|
||||
}
|
||||
if got := answersFirst(argv); got != c.want {
|
||||
t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call.
|
||||
func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil || len(behind) != 0 {
|
||||
t.Fatalf("%v %v", behind, err)
|
||||
}
|
||||
calls, ok := handlers["calls"]
|
||||
if !ok {
|
||||
t.Fatal("calls is not served")
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), `"node"`) {
|
||||
t.Errorf("calls took an argument it does not declare: %v", err)
|
||||
}
|
||||
if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil ||
|
||||
!strings.Contains(err.Error(), "not across a restart") {
|
||||
t.Errorf("an unknown call was not said plainly: %v", err)
|
||||
}
|
||||
got, err := calls(context.Background(), json.RawMessage(`{}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, listed := got.(map[string]any)["calls"]; !listed {
|
||||
t.Errorf("calls answered %v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// moduleCheck judges manifests where they are written, with no mesh (novox/hq ADR 0037, issue 148).
|
||||
//
|
||||
// **The same functions registration runs, and nothing the command line adds** (ADR 0035): the strict
|
||||
// parse with every per-manifest problem, then the rules no single manifest can be judged against,
|
||||
// over exactly the manifests given. Somebody describing their own application in their own
|
||||
// repository runs this before pushing and finds out there, rather than when a running mesh refuses
|
||||
// the registration or, later, when a machine applies something that resolved and should not have.
|
||||
//
|
||||
// **What it cannot know without a store, it says.** The mesh's own seat set is the store's (ADR
|
||||
// 0122); this binary carries a compiled copy that the store overrides when loaded, so a claim on a
|
||||
// mesh seat is judged fully only at registration. A seat another module declares is unknown unless
|
||||
// that module's manifest is passed too. Both are printed as a note, not as a problem — a check that
|
||||
// refused what it could not see would teach people to ignore it.
|
||||
func moduleCheck(paths []string, out io.Writer) error {
|
||||
if len(paths) == 0 {
|
||||
return errors.New("module check <manifest.json>... — one file per module; pass every " +
|
||||
"manifest of a repository together so the rules between them are checked too")
|
||||
}
|
||||
shelf := catalogue.Shelf{}
|
||||
faulted := map[string]bool{}
|
||||
failed := 0
|
||||
for _, path := range paths {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
fmt.Fprintf(out, "%s: %v\n", path, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if first, twice := shelf[m.Module]; twice {
|
||||
_ = first
|
||||
fmt.Fprintf(out, "%s: %s was already given; two manifests name one module\n", path, m.Module)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
// A definition names no installation (novox/hq ADR 0112, ADR 0155): judged here, in the
|
||||
// catalogue-wide test, and at registration, which refuses in the same words.
|
||||
if named := catalogue.InstallationProblems(m); len(named) > 0 {
|
||||
for _, p := range named {
|
||||
fmt.Fprintf(out, "%s: %s\n", path, p)
|
||||
}
|
||||
failed += len(named)
|
||||
faulted[m.Module] = true
|
||||
}
|
||||
shelf[m.Module] = m
|
||||
}
|
||||
|
||||
// Between the manifests: a seat declared twice, a use of a seat nothing declares, a claim on
|
||||
// a seat that does not exist. Run only over what parsed, because a problem inside one manifest
|
||||
// has already been said and would be said again here in a worse form.
|
||||
problems := catalogue.CatalogueProblems(shelf)
|
||||
sort.Strings(problems)
|
||||
for _, p := range problems {
|
||||
fmt.Fprintln(out, p)
|
||||
}
|
||||
failed += len(problems)
|
||||
|
||||
var names []string
|
||||
for name := range shelf {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
m := shelf[name]
|
||||
if faulted[name] {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(out, "%s: ok", name)
|
||||
if n := len(m.Tools); n > 0 {
|
||||
fmt.Fprintf(out, ", %d tool(s)", n)
|
||||
}
|
||||
if len(m.Invokes) > 0 {
|
||||
fmt.Fprintf(out, ", invokes %s", joinInvokes(m.Invokes))
|
||||
}
|
||||
// The state it keeps and reads (novox/hq ADR 0201), so a reviewer sees what lands on the bus.
|
||||
if len(m.State) > 0 {
|
||||
kept := make([]string, 0, len(m.State))
|
||||
for _, s := range m.State {
|
||||
kept = append(kept, s.Name)
|
||||
}
|
||||
fmt.Fprintf(out, ", keeps state %s", strings.Join(kept, ", "))
|
||||
}
|
||||
if len(m.Reads) > 0 {
|
||||
fmt.Fprintf(out, ", reads %s", strings.Join(m.Reads, ", "))
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d problem(s) in %d manifest(s)", failed, len(paths))
|
||||
}
|
||||
fmt.Fprintf(out, "%d manifest(s) checked. Judged against the seats this binary carries; a claim on "+
|
||||
"one of the mesh's own seats is judged fully at registration, and a seat declared by a "+
|
||||
"module not given here reads as unknown\n", len(paths))
|
||||
return nil
|
||||
}
|
||||
|
||||
func joinInvokes(invokes []string) string {
|
||||
if len(invokes) == 1 && invokes[0] == "*" {
|
||||
return "every tool"
|
||||
}
|
||||
s := ""
|
||||
for i, t := range invokes {
|
||||
if i > 0 {
|
||||
s += ", "
|
||||
}
|
||||
s += t
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// manifestsUnder lists every module.json below a directory, for `module check <dir>`.
|
||||
func manifestsUnder(dir string) ([]string, error) {
|
||||
var found []string
|
||||
err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() && (d.Name() == "node_modules" || d.Name() == ".git" || d.Name() == "dist") {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
if !d.IsDir() && d.Name() == "module.json" {
|
||||
found = append(found, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
sort.Strings(found)
|
||||
return found, err
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The check anybody can run is the check registration runs (novox/hq issue 148, ADR 0037): a manifest
|
||||
// with a known fault is named, and one without passes, with no store opened.
|
||||
func TestModuleCheckNamesAFaultAndNeedsNoMesh(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
good := filepath.Join(dir, "good.json")
|
||||
bad := filepath.Join(dir, "bad.json")
|
||||
os.WriteFile(good, []byte(`{"module":"shop","version":"1","tools":["price"],"invokes":["mesh-catalog.catalog_modules"]}`), 0o600)
|
||||
os.WriteFile(bad, []byte(`{"module":"till","version":"1","invokes":["shop"]}`), 0o600)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{good}, &out); err != nil {
|
||||
t.Fatalf("a sound manifest was refused: %v\n%s", err, out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "shop: ok, 1 tool(s), invokes mesh-catalog.catalog_modules") {
|
||||
t.Fatalf("the report does not say what it checked:\n%s", out.String())
|
||||
}
|
||||
|
||||
out.Reset()
|
||||
err := moduleCheck([]string{good, bad}, &out)
|
||||
if err == nil {
|
||||
t.Fatal("a manifest invoking a module and no tool passed")
|
||||
}
|
||||
if !strings.Contains(out.String(), `till invokes "shop", which does not name a tool`) {
|
||||
t.Fatalf("the fault is not named in the manifest's words:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The rules between manifests run over what was given together: a seat two modules declare is
|
||||
// refused, which no single-manifest check can see.
|
||||
func TestModuleCheckJudgesBetweenTheManifestsGiven(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
a := filepath.Join(dir, "a.json")
|
||||
b := filepath.Join(dir, "b.json")
|
||||
os.WriteFile(a, []byte(`{"module":"a","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||
os.WriteFile(b, []byte(`{"module":"b","version":"1","seats":[{"name":"printer","scope":"mesh"}]}`), 0o600)
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck([]string{a, b}, &out); err == nil {
|
||||
t.Fatalf("two declarations of one seat passed:\n%s", out.String())
|
||||
}
|
||||
if !strings.Contains(out.String(), "a seat name means one protocol") {
|
||||
t.Fatalf("the cross-manifest rule was not the one named:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// The real catalogue passes the command, the way it passes the test that used to be the only check.
|
||||
func TestModuleCheckPassesTheCatalogue(t *testing.T) {
|
||||
root := filepath.Join("..", "..", "..", "mesh-catalog", "modules")
|
||||
if _, err := os.Stat(root); err != nil {
|
||||
t.Skipf("catalogue sibling not present: %v", err)
|
||||
}
|
||||
paths, err := manifestsUnder(root)
|
||||
if err != nil || len(paths) == 0 {
|
||||
t.Fatalf("no manifests under %s: %v", root, err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := moduleCheck(paths, &out); err != nil {
|
||||
t.Fatalf("the catalogue does not pass its own check: %v\n%s", err, out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistrationRefusesADefinitionNamingAnInstallation(t *testing.T) {
|
||||
// novox/hq ADR 0155: the check moves to registration once the catalogue passes it. Both
|
||||
// ways in — `module add` and a build's result — go through this, and a name declared on
|
||||
// purpose passes with its reason.
|
||||
named := catalogue.Manifest{Module: "idp", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "image": "x@sha256:aa",
|
||||
"env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}},
|
||||
}}
|
||||
err := namesNoInstallation(named)
|
||||
if err == nil || !strings.Contains(err.Error(), "login.mesh-one.be") ||
|
||||
!strings.Contains(err.Error(), catalogue.NamesOnPurpose) {
|
||||
t.Fatalf("a definition naming an installation is refused with the name and the way out; got %v", err)
|
||||
}
|
||||
meant := catalogue.Manifest{Module: "site", Resources: []map[string]any{
|
||||
{"id": "server", "type": "container", "image": "registry.mesh-one.be/org/site@sha256:cc",
|
||||
catalogue.NamesOnPurpose: map[string]any{
|
||||
"registry.mesh-one.be": "built outside the mesh until its repository is a build source here"}},
|
||||
}}
|
||||
if err := namesNoInstallation(meant); err != nil {
|
||||
t.Fatalf("a name declared on purpose passes; got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Letting the artifact store go of what the mesh no longer keeps (novox/hq ADR 0189, issue 108).
|
||||
//
|
||||
// **Run where the records change.** A build is the moment new bytes landed in the store and the
|
||||
// moment the keep set moved, so it is the moment to say what may go — and it needs no timer of
|
||||
// its own. Reclaiming the bytes is the store's own nightly step; this only decides.
|
||||
//
|
||||
// Never fatal to a build. The build succeeded, the module is registered, and a store that could
|
||||
// not be reached is a thing to say rather than a reason to undo any of that. The next build asks
|
||||
// again, and the references it could not collect are still uncollected, so nothing is lost by
|
||||
// having failed.
|
||||
|
||||
// collect asks the store to let go of everything the mesh made and no longer keeps, and records
|
||||
// what it let go of. Says what it did and what it could not; returns nothing, because nothing
|
||||
// upstream should branch on it.
|
||||
func collect(ctx context.Context, inv *inventory.Inventory) {
|
||||
references, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out what the artifact store may let go of: %v\n", err)
|
||||
return
|
||||
}
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not work out which archives the artifact store keeps: %v\n", err)
|
||||
return
|
||||
}
|
||||
if len(references) == 0 && len(kept) == 0 {
|
||||
return
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read the catalogue to find the artifact store: %v\n", err)
|
||||
return
|
||||
}
|
||||
// As the mesh reaches it from the network. Empty means the store is not on the network — on a
|
||||
// mesh being raised it is not yet, and there the store holds one build of anything and has
|
||||
// nothing to collect.
|
||||
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil || address == "" {
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not find the artifact store to collect from: %v\n", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// **Bounded, because this runs inside somebody's build.** The first sweep of a mesh that has
|
||||
// never collected has the whole history to get through, and a person waiting on `build` should
|
||||
// not pay for it. Two bounds, and what is left over is simply offered again next time —
|
||||
// builds are frequent, and the point is that the store stops growing, not that it empties
|
||||
// tonight.
|
||||
within, stop := context.WithTimeout(ctx, sweepBudget)
|
||||
defer stop()
|
||||
store := artifacts.Store{Address: address}
|
||||
|
||||
// **Hold before letting go** (novox/hq issue 253). The store's collector keeps only what a
|
||||
// manifest names, and archives were published as bare blobs, so every kept archive is first
|
||||
// held by its manifest — which backfills the ones published before holders, a few at a time
|
||||
// as builds come, and is two HEADs each once done. A kept archive that could not be held stops
|
||||
// the sweep before it deletes anything: "everything kept is held" is the precondition the
|
||||
// collector's safety rests on, and a store refusing a hold would refuse the deletes too.
|
||||
wrote, missing, err := holdKept(within, store, kept)
|
||||
if wrote > 0 {
|
||||
fmt.Fprintf(os.Stderr, "the artifact store now holds %d more kept archive(s) by a manifest\n", wrote)
|
||||
}
|
||||
if missing > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d archive(s) the mesh keeps are not in the artifact store at all; "+
|
||||
"`collection` lists them\n", missing)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "not every kept archive could be held, so nothing was let go: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var done []string
|
||||
var left, skipped int
|
||||
for i, reference := range references {
|
||||
if i >= mostPerSweep || within.Err() != nil {
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
err := store.LetGo(within, reference)
|
||||
if err == nil || errors.Is(err, artifacts.Gone) {
|
||||
// Gone is the outcome wanted, already true. Recorded so the next sweep does not ask
|
||||
// again for ever.
|
||||
done = append(done, reference)
|
||||
continue
|
||||
}
|
||||
if errors.Is(err, artifacts.ErrNotOurs) {
|
||||
// **A fact about this record, so this record is skipped** (novox/hq issue 226). Not
|
||||
// marked collected — the mesh did not remove it and should not claim to — and not a
|
||||
// reason to stop, because the store was never asked. One of these at the front of
|
||||
// the oldest-first order ended every sweep until this.
|
||||
skipped++
|
||||
if skipped == 1 {
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"the sweep will not address %s and went on: %v\n", reference, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// **Stopped at the first refusal by the STORE, not pushed through.** A store that refuses
|
||||
// one refuses all of them — deletion disabled, the store down, the network gone — so
|
||||
// going on would be a hundred identical failures and a hundred identical log lines in
|
||||
// front of whoever was building something.
|
||||
fmt.Fprintf(os.Stderr, "the artifact store kept %s, so nothing more was asked of it: %v\n",
|
||||
reference, err)
|
||||
left = len(references) - i
|
||||
break
|
||||
}
|
||||
|
||||
if len(done) > 0 {
|
||||
// Recorded outside `within`: the deletions happened, and losing the record of them because
|
||||
// the sweep ran out of budget would mean asking about them again for ever.
|
||||
if err := inv.MarkCollected(ctx, done); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "the store let go of %d artifact(s) and the record of it did not keep: %v\n",
|
||||
len(done), err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "the artifact store let go of %d artifact(s) the mesh no longer keeps\n",
|
||||
len(done))
|
||||
}
|
||||
if left > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d more to collect; the next build asks again\n", left)
|
||||
}
|
||||
if skipped > 0 {
|
||||
fmt.Fprintf(os.Stderr, "%d artifact(s) the sweep will not address were skipped\n", skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// holdKept holds every kept archive by its manifest, stopping at the first refusal by the store.
|
||||
// Answers how many holders it wrote and how many kept archives the store does not have.
|
||||
//
|
||||
// A missing archive is counted rather than fatal: there is nothing to hold, and that is a fact
|
||||
// for an operator to read (`collection`), not a reason to stop collecting what is not kept. A
|
||||
// reference the store cannot be asked about is skipped as the deletion loop skips one
|
||||
// (novox/hq issue 226).
|
||||
func holdKept(ctx context.Context, store artifacts.Store, kept []string) (wrote, missing int, err error) {
|
||||
for _, reference := range kept {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return wrote, missing, fmt.Errorf("ran out of time before %s: %w", reference, err)
|
||||
}
|
||||
did, err := store.Hold(ctx, reference)
|
||||
switch {
|
||||
case err == nil:
|
||||
if did {
|
||||
wrote++
|
||||
}
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
missing++
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
default:
|
||||
return wrote, missing, fmt.Errorf("holding %s: %w", reference, err)
|
||||
}
|
||||
}
|
||||
return wrote, missing, nil
|
||||
}
|
||||
|
||||
// mostPerSweep is how many artifacts one sweep will ask about. Enough that a mesh building
|
||||
// several times a day converges within days of this landing; small enough that no single build
|
||||
// waits on the whole backlog.
|
||||
const mostPerSweep = 200
|
||||
|
||||
// sweepBudget is the longest a sweep will keep a build waiting.
|
||||
const sweepBudget = 60 * time.Second
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/artifacts"
|
||||
)
|
||||
|
||||
// What the artifact store keeps, whether each kept archive is held, and what the sweep may let go
|
||||
// (novox/hq issue 253, ADR 0189).
|
||||
//
|
||||
// **The question to answer before the store's collector runs for real.** The collector deletes
|
||||
// every blob no manifest names, and archives were published as bare blobs, so the nightly step
|
||||
// runs `--dry-run` until every archive the mesh keeps is held by its manifest. The sweep holds
|
||||
// them as builds come; this says how far that has got — "0 unheld" is the number that lets the
|
||||
// dry run go.
|
||||
//
|
||||
// Reads and changes nothing: each kept archive is asked about with HEADs only. Reached over the
|
||||
// console through the mesh-controller seat's `command` verb (`collection --json`), which needs no
|
||||
// new verb in the seat's row.
|
||||
|
||||
type collectionReport struct {
|
||||
// Store is the artifact store as this machine reached it; empty when it is not on the network.
|
||||
Store string `json:"store"`
|
||||
// KeptArchives is how many archives the mesh keeps, for either reason.
|
||||
KeptArchives int `json:"kept_archives"`
|
||||
// Held is how many of them the store holds by their manifest.
|
||||
Held int `json:"held"`
|
||||
// Unheld are the kept archives the collector would delete tonight if it ran for real.
|
||||
Unheld []string `json:"unheld"`
|
||||
// Missing are kept archives the store does not have at all.
|
||||
Missing []string `json:"missing"`
|
||||
// Unasked is how many could not be asked about, and why the asking stopped.
|
||||
Unasked int `json:"unasked"`
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
// Eligible is what the sweep may let go of: made by the mesh, kept for no reason, not yet
|
||||
// collected — split by kind.
|
||||
Eligible int `json:"eligible"`
|
||||
EligibleImages int `json:"eligible_images"`
|
||||
EligibleArchives int `json:"eligible_archives"`
|
||||
// SafeToCollect is whether every kept archive was asked about and every one is held.
|
||||
SafeToCollect bool `json:"safe_to_collect"`
|
||||
}
|
||||
|
||||
func collectionCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("collection", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "answer as JSON")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 0 {
|
||||
return errors.New("collection [--json]")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
kept, err := inv.KeptArchives(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
eligible, err := inv.ToCollect(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report := collectionReport{KeptArchives: len(kept), Eligible: len(eligible), Unheld: []string{}, Missing: []string{}}
|
||||
for _, reference := range eligible {
|
||||
if strings.Contains(reference, "/blobs/") {
|
||||
report.EligibleArchives++
|
||||
} else {
|
||||
report.EligibleImages++
|
||||
}
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.Store, err = artifactStoreAddress(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if report.Store == "" {
|
||||
report.Unasked = len(kept)
|
||||
report.Stopped = "this mesh has no artifact store on its network"
|
||||
} else {
|
||||
report.Unasked, report.Stopped = askHeld(ctx, artifacts.Store{Address: report.Store}, kept, &report)
|
||||
}
|
||||
report.SafeToCollect = report.Unasked == 0 && len(report.Unheld) == 0
|
||||
|
||||
if *asJSON {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
printCollection(report)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askHeld asks the store about each kept archive, stopping at the first answer that is not about
|
||||
// the archive: a store that cannot be reached for one cannot be for the next, and a page of
|
||||
// identical failures says less than one line.
|
||||
func askHeld(ctx context.Context, store artifacts.Store, kept []string, report *collectionReport) (int, string) {
|
||||
for i, reference := range kept {
|
||||
held, err := store.Held(ctx, reference)
|
||||
switch {
|
||||
case err == nil && held:
|
||||
report.Held++
|
||||
case err == nil:
|
||||
report.Unheld = append(report.Unheld, reference)
|
||||
case errors.Is(err, artifacts.Gone):
|
||||
report.Missing = append(report.Missing, reference)
|
||||
case errors.Is(err, artifacts.ErrNotOurs):
|
||||
// KeptArchives names only the mesh's own; counted as unasked if one ever is not.
|
||||
report.Unasked++
|
||||
default:
|
||||
return report.Unasked + len(kept) - i, fmt.Sprintf("asking about %s: %v", reference, err)
|
||||
}
|
||||
}
|
||||
return report.Unasked, ""
|
||||
}
|
||||
|
||||
func printCollection(r collectionReport) {
|
||||
store := r.Store
|
||||
if store == "" {
|
||||
store = "(not on the network)"
|
||||
}
|
||||
fmt.Printf("artifact store %s\n", store)
|
||||
fmt.Printf("kept archives %d\n", r.KeptArchives)
|
||||
fmt.Printf(" held %d\n", r.Held)
|
||||
fmt.Printf(" unheld %d\n", len(r.Unheld))
|
||||
fmt.Printf(" missing %d\n", len(r.Missing))
|
||||
if r.Unasked > 0 {
|
||||
fmt.Printf(" not asked %d (%s)\n", r.Unasked, r.Stopped)
|
||||
}
|
||||
fmt.Printf("eligible to let go %d (%d images, %d archives)\n", r.Eligible, r.EligibleImages, r.EligibleArchives)
|
||||
if len(r.Unheld) > 0 {
|
||||
fmt.Println("\nunheld — the store's collector would delete these; the next build's sweep holds them:")
|
||||
for _, reference := range r.Unheld {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
if len(r.Missing) > 0 {
|
||||
fmt.Println("\nmissing — kept by the mesh, not in the store:")
|
||||
for _, reference := range r.Missing {
|
||||
fmt.Printf(" %s\n", reference)
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
if r.SafeToCollect {
|
||||
fmt.Println("every kept archive is held: the store's collector may run for real")
|
||||
} else {
|
||||
fmt.Println("NOT every kept archive is known to be held: keep the store's collector on --dry-run")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A fresh assignment is pushed before its credential exists (novox/hq issue 203): `assign` recorded
|
||||
// the module, `push` sealed a random own secret where the bus credential belongs, and the process
|
||||
// crash-looped until a person ran `module issue` and pushed again. Now assigning a module that speaks
|
||||
// on the bus issues its credential in the same act — or, when the bus cannot be reached from here,
|
||||
// says which verb to run — and a push never seals a placeholder in a credential's place.
|
||||
|
||||
func aTalker() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "talker", Version: "1",
|
||||
OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/talker/broker"}},
|
||||
Resources: []map[string]any{
|
||||
{"id": "state", "type": "directory", "path": "/var/lib/mesh/talker", "mode": "0700"},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAssigningAModuleThatSpeaksOnTheBusNamesItsCredential(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aTalker())
|
||||
|
||||
// No bus is known to this process, so the credential cannot be issued here: the assignment
|
||||
// stands and says exactly what must happen before a push — never silently.
|
||||
said, err := assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "module issue talker --node laptop") {
|
||||
t.Fatalf("an assignment whose credential could not be issued does not name the verb:\n%s", said)
|
||||
}
|
||||
|
||||
// And the push refuses to send it, naming the same verb, rather than sealing a placeholder.
|
||||
plan, settings, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err == nil {
|
||||
t.Fatal("a push sealed a placeholder where talker's bus credential belongs")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "module issue talker --node laptop") || !strings.Contains(err.Error(), "issue 203") {
|
||||
t.Fatalf("the refusal does not say what to run: %v", err)
|
||||
}
|
||||
|
||||
// Once the user is minted, the push goes on to the credential the mesh sealed, and re-assigning
|
||||
// does not mint again: a credential rotates on purpose, never by habit.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "laptop.talker", Kind: inventory.BusModule, Node: "laptop", Module: "talker"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hash, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err = assign(ctx, open, "laptop", "talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "module issue") {
|
||||
t.Fatalf("a module with a minted credential was told to issue one:\n%s", said)
|
||||
}
|
||||
again, _, err := open.inventory.BusUserHash(ctx, "laptop.talker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again != hash {
|
||||
t.Fatal("re-assigning rotated the credential")
|
||||
}
|
||||
}
|
||||
|
||||
// A module that declares no broker secret is left alone: nothing to issue, nothing said.
|
||||
func TestAssigningAModuleThatDoesNotSpeakSaysNothingOfCredentials(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
register(t, open, helloWeb())
|
||||
said, err := assign(t.Context(), open, "laptop", "hello-web")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(said, "credential") {
|
||||
t.Fatalf("a module without a broker secret was told about credentials:\n%s", said)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// recordingDelivery is a delivery that writes down what was done, in order, and fails where told.
|
||||
type recordingDelivery struct {
|
||||
did []string
|
||||
grantErr error
|
||||
declareErr error
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error {
|
||||
for _, s := range sending {
|
||||
r.did = append(r.did, "grant "+s.node)
|
||||
}
|
||||
return r.grantErr
|
||||
}
|
||||
|
||||
func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) {
|
||||
if r.declareErr != nil {
|
||||
return "", r.declareErr
|
||||
}
|
||||
r.did = append(r.did, "declare "+s.node)
|
||||
return "digest-" + s.node, nil
|
||||
}
|
||||
|
||||
func ready(names ...string) []readyNode {
|
||||
var out []readyNode
|
||||
for _, n := range names {
|
||||
out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq issue 249: the grants that come with a module's new declarations are issued before any
|
||||
// machine is sent the code that uses them — except the machine holding the bus, whose declaration
|
||||
// carries the controller's own right to issue them, and goes first.
|
||||
func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) {
|
||||
d := &recordingDelivery{}
|
||||
digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"}
|
||||
if !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered in the order %v, wanted %v", d.did, want)
|
||||
}
|
||||
if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" {
|
||||
t.Fatalf("the digests sent were not answered: %v", digests)
|
||||
}
|
||||
|
||||
held := &recordingDelivery{}
|
||||
if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"}
|
||||
if !reflect.DeepEqual(held.did, want) {
|
||||
t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A grant that cannot be issued holds back the machines it concerns and is an error the caller
|
||||
// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the
|
||||
// grant that would let the controller issue memberships is never held behind them.
|
||||
func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) {
|
||||
// A failure naming no machine (the buckets): everything but the bus's machine.
|
||||
d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")}
|
||||
_, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop"))
|
||||
if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") ||
|
||||
!strings.Contains(err.Error(), "anchor, laptop not sent") {
|
||||
t.Fatalf("a failed grant was not said as the send's failure: %v", err)
|
||||
}
|
||||
if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) {
|
||||
t.Fatalf("delivered %v, wanted the bus's machine alone", d.did)
|
||||
}
|
||||
|
||||
// A membership that failed for one machine: that machine alone.
|
||||
one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}}
|
||||
_, err = deliver(t.Context(), one, "", ready("anchor", "laptop"))
|
||||
if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") {
|
||||
t.Fatalf("one machine's refused membership was not said: %v", err)
|
||||
}
|
||||
if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) {
|
||||
t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did)
|
||||
}
|
||||
|
||||
// The announced upgrade that hit it is asked again.
|
||||
if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) {
|
||||
t.Fatal("an announcement whose send stopped at its grants is not asked again")
|
||||
}
|
||||
if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) {
|
||||
t.Fatal("any failure is asked again, not only a grant's")
|
||||
}
|
||||
|
||||
// Nothing to send is nothing granted either.
|
||||
none := &recordingDelivery{grantErr: errors.New("never asked")}
|
||||
if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 {
|
||||
t.Fatalf("an empty send granted or failed: %v %v", none.did, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Whether the bus's machine goes first is read from the user list alone, by its digest.
|
||||
func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) {
|
||||
list := "users: [a, b]"
|
||||
if userListBehind(list, digestOf([]byte(list))) {
|
||||
t.Fatal("the list it was sent reads as behind")
|
||||
}
|
||||
if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") {
|
||||
t.Fatal("a changed or never-sent list reads as current")
|
||||
}
|
||||
if userListBehind("", "") {
|
||||
t.Fatal("a machine sent no list reads as behind")
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus goes first: its declaration carries the user list the new grants are
|
||||
// checked against. Among the machines it is moved to the front; not among them it is added only
|
||||
// when it is behind.
|
||||
func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
names []string
|
||||
holder string
|
||||
behind bool
|
||||
want []string
|
||||
}{
|
||||
{"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}},
|
||||
{"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}},
|
||||
{"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}},
|
||||
{"only it", []string{"novox"}, "novox", false, []string{"novox"}},
|
||||
} {
|
||||
if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
|
||||
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
|
||||
// module built on one of those moves as well.
|
||||
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
|
||||
entry := func(name string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
|
||||
}
|
||||
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
|
||||
against := map[string][]string{
|
||||
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
|
||||
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
|
||||
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
|
||||
}
|
||||
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
|
||||
var names []string
|
||||
for _, e := range got {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
|
||||
if len(names) != len(want) {
|
||||
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
|
||||
}
|
||||
for _, n := range names {
|
||||
if !want[n] {
|
||||
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
|
||||
}
|
||||
}
|
||||
// The dependents come in base order when the merge orders them: the runtime, then shop, then
|
||||
// the plugin that stands on shop.
|
||||
ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against)
|
||||
pos := map[string]int{}
|
||||
for i, e := range ordered {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) {
|
||||
t.Fatalf("not in base order: %v", ordered)
|
||||
}
|
||||
// Nothing moved: nothing follows.
|
||||
if more := dependentsOf(nil, entries, against); len(more) != 0 {
|
||||
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no
|
||||
// store at all, so anything that tried to record or register would fail rather than pass quietly.
|
||||
func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) {
|
||||
err := builds{}.Built(t.Context(), link.BuildResult{
|
||||
ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true,
|
||||
Manifest: json.RawMessage(`{"module":"app","version":"1"}`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("a dry run's outcome was not simply set aside: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The mark survives the wire both ways: asked as a dry run, answered as one.
|
||||
func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) {
|
||||
raw, _ := json.Marshal(link.BuildRequest{ID: "build-1", Repository: "r", DryRun: true})
|
||||
var asked link.BuildRequest
|
||||
if err := json.Unmarshal(raw, &asked); err != nil || !asked.DryRun {
|
||||
t.Fatalf("the request lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-1", DryRun: true})
|
||||
var answered link.BuildResult
|
||||
if err := json.Unmarshal(raw, &answered); err != nil || !answered.DryRun {
|
||||
t.Fatalf("the outcome lost its dry-run mark: %s", raw)
|
||||
}
|
||||
raw, _ = json.Marshal(link.BuildResult{ID: "build-2"})
|
||||
if string(raw) != `{"id":"build-2","repository":"","on":""}` {
|
||||
t.Fatalf("an ordinary outcome carries a dry-run mark: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq
|
||||
// issue 259, ADR 0221).
|
||||
//
|
||||
// A named push ends by sending every other machine whose declaration differs from what it was last
|
||||
// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A
|
||||
// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls
|
||||
// out makes every machine running it differ from the merge on, and so did a module whose plan was
|
||||
// still waiting on its first machine (ADR 0218): `push <anchor>` sent all four machines the build
|
||||
// that was meant to be walked through the mesh one machine at a time, and a fault in it was met
|
||||
// everywhere at once.
|
||||
//
|
||||
// What tells the two apart is which build of each module the machine was last sent, kept with every
|
||||
// send. A machine any of whose modules would move to a build its policy or an open plan holds back
|
||||
// is not sent by a push that did not name it; the push says which, and why, and how to send it.
|
||||
|
||||
// heldBack is why a push that did not name a machine must not send it, empty when it may.
|
||||
//
|
||||
// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as
|
||||
// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the
|
||||
// machine was last sent — including a module the machine was never sent at all. A move is held when
|
||||
// the module's policy records rather than rolls out, or when an open plan has not yet sent this
|
||||
// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried
|
||||
// is not known, so a held upgrade cannot be told from anything else.
|
||||
func heldBack(node string, modules []string, sent map[string]string, known bool,
|
||||
current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string {
|
||||
if !known {
|
||||
return []string{"which builds it was last sent is not known — it was last sent before the " +
|
||||
"mesh kept them, or sent a declaration by hand"}
|
||||
}
|
||||
var why []string
|
||||
for _, m := range modules {
|
||||
now := current[m]
|
||||
was, carried := sent[m]
|
||||
if carried && was == now.Commit {
|
||||
continue
|
||||
}
|
||||
move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit))
|
||||
if !now.RollOut {
|
||||
why = append(why, move+", which its upgrade policy records rather than rolls out")
|
||||
continue
|
||||
}
|
||||
if id := planStillToSend(plans, m, node); id != "" {
|
||||
why = append(why, move+", which "+id+" has not sent it yet (one machine first)")
|
||||
}
|
||||
}
|
||||
sort.Strings(why)
|
||||
return why
|
||||
}
|
||||
|
||||
// planStillToSend is the open plan that has a module's new build still to send this machine, or empty:
|
||||
// one holding the module that has neither finished sending it nor sent it here first, and has not
|
||||
// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made
|
||||
// per machine.
|
||||
func planStillToSend(plans []inventory.Plan, module, node string) string {
|
||||
for _, p := range plans {
|
||||
s, holds := p.Modules[module]
|
||||
if !p.Open() || !holds {
|
||||
continue
|
||||
}
|
||||
if s == nil {
|
||||
return p.ID
|
||||
}
|
||||
if s.SentAt != nil || s.State == "failed" {
|
||||
continue
|
||||
}
|
||||
first := false
|
||||
for _, n := range s.First {
|
||||
if n == node {
|
||||
first = true
|
||||
}
|
||||
}
|
||||
if !first {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func fromBuild(was string, carried bool) string {
|
||||
if !carried {
|
||||
return "(never sent it) "
|
||||
}
|
||||
return "from " + buildName(was) + " "
|
||||
}
|
||||
|
||||
func buildName(commit string) string {
|
||||
if commit == "" {
|
||||
return "a build with no source"
|
||||
}
|
||||
return shortCommit(commit)
|
||||
}
|
||||
|
||||
// heldMachines reads, for each machine named, why a push that did not name it must not send it
|
||||
// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left
|
||||
// to the send, which says why.
|
||||
func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
if len(names) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
inv := open.inventory
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
modules := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
modules = append(modules, m.Module)
|
||||
}
|
||||
sent, known, err := inv.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if why := heldBack(node, modules, sent, known, current, plans); len(why) > 0 {
|
||||
out[node] = why
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it
|
||||
// was not sent, that whatever else it is owed waits with it, and the command that sends it.
|
||||
func sayHeld(w io.Writer, node string, why []string) {
|
||||
fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+
|
||||
"holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+
|
||||
"grant from this push among it — waits with it. `push %s` sends it\n",
|
||||
node, strings.Join(why, "; "), node)
|
||||
}
|
||||
|
||||
// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over
|
||||
// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose
|
||||
// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221).
|
||||
//
|
||||
// `handled` is every machine already sent or already said; it is not considered again. Answers the
|
||||
// machines that could not be composed, as refusals.
|
||||
func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool,
|
||||
compose func(held context.Context, node string) (sendable, error), d delivery, holder string,
|
||||
w io.Writer) ([]string, error) {
|
||||
inv := open.inventory
|
||||
var refusals []string
|
||||
// Bounded by the node count: a node is marked handled the round it is considered and is never
|
||||
// considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard
|
||||
// against a logic error, not a real limit — if it were ever hit, that is a bug rather than a
|
||||
// cascade legitimately still converging, so it is said rather than passed over in silence.
|
||||
rounds := 0
|
||||
for {
|
||||
would, err := wouldSend(ctx, open, nodes)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
behind, err := inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var also []string
|
||||
for _, m := range behind {
|
||||
if !handled[m.Node] {
|
||||
also = append(also, m.Node)
|
||||
}
|
||||
}
|
||||
if len(also) == 0 {
|
||||
return refusals, nil
|
||||
}
|
||||
if rounds++; rounds > len(nodes) {
|
||||
fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+
|
||||
"should not happen; run `push --behind` to finish\n",
|
||||
rounds-1, strings.Join(also, ", "))
|
||||
return refusals, nil
|
||||
}
|
||||
sort.Strings(also)
|
||||
held, err := heldMachines(ctx, open, also)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
var sending []string
|
||||
for _, name := range also {
|
||||
// Every candidate this round is marked handled — the sent ones so they are not
|
||||
// re-listed, the held ones because they stay held, and the refused ones so a machine
|
||||
// that cannot be composed does not make the loop spin on it for ever.
|
||||
handled[name] = true
|
||||
if why, isHeld := held[name]; isHeld {
|
||||
sayHeld(w, name, why)
|
||||
continue
|
||||
}
|
||||
sending = append(sending, name)
|
||||
}
|
||||
if len(sending) == 0 {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+
|
||||
"declaration since changed; sending it too\n", strings.Join(sending, ", "))
|
||||
// Tolerantly, exactly as the named send: a machine that cannot be composed is collected as
|
||||
// a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066).
|
||||
// Held for this round only, and after the last round's were given back, so two pushes
|
||||
// cascading into each other's machines never each wait on the other.
|
||||
refused, err := sendRound(ctx, open, sending, compose, d, holder)
|
||||
refusals = append(refusals, refused...)
|
||||
if err != nil {
|
||||
return refusals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what
|
||||
// is left out of it said, and its declaration allocated.
|
||||
func composeForPush(open *stores, gens map[string]catalogue.Generator) func(held context.Context, node string) (sendable, error) {
|
||||
return func(held context.Context, node string) (sendable, error) {
|
||||
plan, settings, err := planFor(held, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
reportUnhostable(node, plan)
|
||||
declared, err := declarationWith(held, open, node, plan, settings, gens, Allocating)
|
||||
if err == nil {
|
||||
reportLeftOut(node, declared)
|
||||
}
|
||||
return declared, err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
||||
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
||||
// else that moved is still a consequence the push sends (ADR 0083).
|
||||
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{
|
||||
"resolver": {Commit: "c2c2c2c2c2"},
|
||||
"agent": {Commit: "a2", RollOut: true},
|
||||
"network": {},
|
||||
}
|
||||
modules := []string{"network", "resolver", "agent"}
|
||||
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
||||
|
||||
// A module whose policy records moved: held, naming it, both builds and why.
|
||||
why := heldBack("laptop", modules, sent, true, current, nil)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
||||
!strings.Contains(why[0], "upgrade policy records") {
|
||||
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
||||
}
|
||||
|
||||
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
||||
sent["resolver"] = "c2c2c2c2c2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
||||
}
|
||||
|
||||
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
||||
sent["agent"] = "a1"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
||||
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
||||
}
|
||||
|
||||
// The last send's builds are not known: held whole.
|
||||
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "not known") {
|
||||
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
||||
}
|
||||
|
||||
// A module the machine was never sent, under a recording policy: held, and said so.
|
||||
delete(sent, "resolver")
|
||||
sent["agent"] = "a2"
|
||||
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
||||
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
||||
t.Fatalf("a module never sent under a recording policy: %v", why)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
||||
// naming some other machine must not send them for it.
|
||||
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
||||
at := time.Now()
|
||||
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
||||
sent := map[string]string{"agent": "a1"}
|
||||
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
||||
|
||||
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
||||
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
||||
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
||||
}
|
||||
// The first machine itself was sent by the plan: not held by it.
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
||||
t.Fatalf("the plan's first machine was held: %v", why)
|
||||
}
|
||||
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
||||
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
||||
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
||||
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
||||
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
||||
t.Errorf("%s: not held: %v", what, why)
|
||||
}
|
||||
}
|
||||
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
||||
for what, plans := range map[string][]inventory.Plan{
|
||||
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
||||
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "failed"}}}},
|
||||
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"}}}},
|
||||
} {
|
||||
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
||||
t.Errorf("%s: held: %v", what, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A send records the build of each module it carried; a module left out of it keeps the build it
|
||||
// was last sent, since the machine keeps that one.
|
||||
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
||||
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
||||
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
||||
current, map[string]string{"a": "a1", "b": "b1"})
|
||||
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
||||
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
||||
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("carried %v, wanted %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
||||
// reads the machine as current — and writes down which machines it declared.
|
||||
type recordedDelivery struct {
|
||||
inv *inventory.Inventory
|
||||
declared []string
|
||||
}
|
||||
|
||||
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
|
||||
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
r.declared = append(r.declared, s.node)
|
||||
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds)
|
||||
}
|
||||
|
||||
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
||||
// says which build it is.
|
||||
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
||||
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
||||
}}
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// A third machine on the private network: once it is sent, every other machine's peers change with
|
||||
// it, which is a consequence a push must still send — no build moved.
|
||||
func aThirdMachine(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.AddNode(ctx, "spare")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
||||
{"name": "systemd", "present": true}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
||||
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
||||
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asked := time.Now().Add(-time.Hour)
|
||||
aResolver(t, open, "c1c1c1c1c1", asked)
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
||||
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
||||
}
|
||||
digestOfLaptop := func() string {
|
||||
sent, err := inv.Outstanding(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return sent
|
||||
}
|
||||
before := digestOfLaptop()
|
||||
|
||||
// The change merges; the policy is the default, record. `push anchor` sends the anchor...
|
||||
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ...and its cascade leaves the laptop, saying so.
|
||||
var said bytes.Buffer
|
||||
d.declared = nil
|
||||
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", &said)
|
||||
if err != nil || len(refused) != 0 {
|
||||
t.Fatalf("the cascade failed: %v %v", refused, err)
|
||||
}
|
||||
if len(d.declared) != 0 {
|
||||
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
||||
}
|
||||
if digestOfLaptop() != before {
|
||||
t.Fatal("the laptop's last send moved: it was sent the held build")
|
||||
}
|
||||
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
||||
"upgrade policy records", "`push laptop` sends it"} {
|
||||
if !strings.Contains(said.String(), want) {
|
||||
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
||||
// is held, and that what else it is owed waits with it.
|
||||
aThirdMachine(t, open)
|
||||
d.declared = nil
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(d.declared) != 0 || digestOfLaptop() != before {
|
||||
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
||||
}
|
||||
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
||||
t.Fatalf("the push did not say both:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A policy that rolls out: the laptop is a consequence like any other, and sent.
|
||||
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
||||
compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
||||
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
||||
t.Fatalf("the new send did not record the new build: %v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
||||
// for another reason is sent by a push that names someone else.
|
||||
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
||||
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
compose := composeForPush(open, gens)
|
||||
d := &recordedDelivery{inv: inv}
|
||||
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// `push spare`, the machine just placed: the others' peers change with it.
|
||||
aThirdMachine(t, open)
|
||||
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
var said bytes.Buffer
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
||||
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
if strings.Contains(said.String(), "was not sent") {
|
||||
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
||||
}
|
||||
|
||||
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d.declared = nil
|
||||
said.Reset()
|
||||
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
||||
// question.
|
||||
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
||||
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||
// through sendTo does not wait on itself.
|
||||
type heldKey struct{}
|
||||
|
||||
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||
// Release gives back only what this call took.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||
var take []string
|
||||
for _, n := range names {
|
||||
if !already[n] {
|
||||
take = append(take, n)
|
||||
}
|
||||
}
|
||||
if len(take) == 0 {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
release, err := open.inventory.HoldNodes(ctx, take)
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for n := range already {
|
||||
held[n] = true
|
||||
}
|
||||
for _, n := range take {
|
||||
held[n] = true
|
||||
}
|
||||
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled
|
||||
// and a send that fails leave nobody waiting for the node.
|
||||
func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
unmarshallable := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil
|
||||
}
|
||||
plain := func(context.Context, string) (sendable, error) {
|
||||
return sendable{Resources: []map[string]any{{"id": "x"}}}, nil
|
||||
}
|
||||
failing := &recordingDelivery{declareErr: errors.New("the broker went away")}
|
||||
fine := &recordingDelivery{}
|
||||
|
||||
for name, round := range map[string]func() error{
|
||||
"a body that cannot be marshalled": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "")
|
||||
return err
|
||||
},
|
||||
"a send that fails": func() error {
|
||||
_, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "")
|
||||
return err
|
||||
},
|
||||
} {
|
||||
if err := round(); err == nil {
|
||||
t.Fatalf("%s was not an error", name)
|
||||
}
|
||||
waiting, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
release, err := open.inventory.HoldNodes(waiting, []string{"anchor"})
|
||||
cancel()
|
||||
if err != nil {
|
||||
t.Fatalf("after %s the node is still held: %v", name, err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// recordDerivedHolders writes down who holds each mesh-scoped seat that nobody was ever recorded
|
||||
// as holding.
|
||||
//
|
||||
// **A seat held by derivation is a seat held by accident of being alone** (novox/hq
|
||||
// 04-ISSUES/170). ADR 0131 lets a holder on record settle a seat, and lets any other assignment
|
||||
// whose module could hold it stand beside the holder, eligible and silent. But a seat nobody
|
||||
// ever handed over has no record, so its holder is whichever assignment happened to be the sole
|
||||
// claimant — and the day a second one is assigned, both claim, both are refused, and the first
|
||||
// one's whole machine stops resolving. That is what assigning a second postgres did to the
|
||||
// control plane's own store.
|
||||
//
|
||||
// So the mesh writes the derived answer down before it acts on an assignment: for every
|
||||
// mesh-scoped seat with exactly one resolved holder and nothing on record, that holder is
|
||||
// recorded as the standing one — the same record `seat <name> --to <node>/<module>` makes by
|
||||
// hand, made from what the mesh already resolved. A seat with two derived claimants is left
|
||||
// alone: that is the ambiguity a person settles, and recording either would be guessing.
|
||||
//
|
||||
// Node-scoped seats are untouched: a record is one holder per seat, and a node-scoped seat has
|
||||
// one holder per machine (ADR 0121), so there is nothing for a record to settle there.
|
||||
func recordDerivedHolders(ctx context.Context, open *stores) ([]string, error) {
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// exclude nobody: every node's claims, resolved with the holdings on record.
|
||||
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A record is a row against a seat the store knows. A seat it does not — a mesh whose seats
|
||||
// were never seeded, a seat a module declares for itself — stays held by derivation, as it
|
||||
// always was; a missing row is not a reason an assignment fails.
|
||||
known, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
recordable := map[string]bool{}
|
||||
for _, s := range known {
|
||||
recordable[s.Name] = true
|
||||
}
|
||||
onRecord := map[string]bool{}
|
||||
for _, h := range recorded {
|
||||
if s, ok := catalogue.SeatNamed(h.Claim); ok {
|
||||
onRecord[s.Name] = true
|
||||
}
|
||||
}
|
||||
holders := map[string][]catalogue.Held{}
|
||||
for _, h := range world.Held {
|
||||
if h.Scope != catalogue.ScopeMesh {
|
||||
continue
|
||||
}
|
||||
s, ok := catalogue.SeatNamed(h.Claim)
|
||||
if !ok || onRecord[s.Name] || !recordable[s.Name] {
|
||||
continue
|
||||
}
|
||||
holders[s.Name] = append(holders[s.Name], h)
|
||||
}
|
||||
names := make([]string, 0, len(holders))
|
||||
for name := range holders {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
var said []string
|
||||
for _, name := range names {
|
||||
if len(holders[name]) != 1 {
|
||||
continue
|
||||
}
|
||||
h := holders[name][0]
|
||||
if err := inv.HoldSeat(ctx, name, catalogue.ScopeMesh, h.Node, h.Module); err != nil {
|
||||
return said, err
|
||||
}
|
||||
said = append(said, fmt.Sprintf(
|
||||
"recorded %s on %s as the standing holder of %s, which it held only by being alone",
|
||||
h.Module, h.Node, name))
|
||||
}
|
||||
return said, nil
|
||||
}
|
||||
|
||||
// replicatedHolders is, for each replicated mesh seat, every machine on the private network holding
|
||||
// it — by internal name, at its private address (novox/hq ADR 0223). What a machine's resolver file
|
||||
// lists for `mesh-dns-resolver`; the rendering puts the machine itself first when it is one.
|
||||
//
|
||||
// **The holders on record, and only the sole claimant when there are none** — the same answer the
|
||||
// resolver gives about who holds (ADR 0131, issue 170). An assignment standing beside the holders,
|
||||
// eligible and silent, is not listed: it becomes a holder by `seat <name> --add`, an act, never by
|
||||
// being assigned. Two claimants with nothing on record are refused at resolution, so neither is
|
||||
// listed here. A holder off the private network is left out: a resolver named at an address nothing
|
||||
// answers is a lookup that waits out its timeout on every name.
|
||||
func replicatedHolders(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]map[string]string, error) {
|
||||
var replicated []catalogue.Seat
|
||||
for _, s := range catalogue.Seats() {
|
||||
if s.Replicated && s.Scope == catalogue.ScopeMesh {
|
||||
replicated = append(replicated, s)
|
||||
}
|
||||
}
|
||||
if len(replicated) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
recorded, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address := map[string]string{}
|
||||
for _, p := range places {
|
||||
address[p.Name] = p.Address
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]map[string]string{}
|
||||
for _, seat := range replicated {
|
||||
var nodes []string
|
||||
for _, h := range recorded {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name && h.Scope == seat.Scope {
|
||||
nodes = append(nodes, h.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
var derived []string
|
||||
for _, e := range entries {
|
||||
for _, c := range e.Manifest.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
derived = append(derived, e.On...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(derived) == 1 {
|
||||
nodes = derived
|
||||
}
|
||||
}
|
||||
at := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
if address[n] != "" {
|
||||
at[overlay.InternalName(n)] = address[n]
|
||||
}
|
||||
}
|
||||
out[seat.Name] = at
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// A seat nobody ever handed over is held by whichever assignment happened to be alone — and the
|
||||
// day a second module able to hold it is assigned, both claimed, both were refused, and the first
|
||||
// one's machine stopped resolving (novox/hq 04-ISSUES/170). The mesh now writes the derived holder
|
||||
// down before it acts, so the second assignment stands beside the holder on record.
|
||||
|
||||
func aSeatedStore() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"postgres-database": {"port": 5432}},
|
||||
Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}}
|
||||
}
|
||||
|
||||
func TestASecondEligibleHolderStandsBesideTheOneHeldByBeingAlone(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
// Every deploy seeds the mesh's own seats; a record is a row against one of them.
|
||||
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, aSeatedStore())
|
||||
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "store")
|
||||
if err != nil {
|
||||
t.Fatalf("a second store, eligible for the seat, was refused:\n%s\n%v", said, err)
|
||||
}
|
||||
if strings.Contains(said, "cannot be worked out") {
|
||||
t.Fatalf("assigning a second store unsettled the first one's machine:\n%s", said)
|
||||
}
|
||||
if !strings.Contains(said, "recorded store on anchor as the standing holder of mesh-store") {
|
||||
t.Fatalf("the holder by derivation was not written down:\n%s", said)
|
||||
}
|
||||
|
||||
holdings, err := open.inventory.Holdings(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, h := range holdings {
|
||||
if h.Claim == "mesh-store" {
|
||||
found = true
|
||||
if h.Node != "anchor" || h.Module != "store" {
|
||||
t.Fatalf("mesh-store is recorded on %s/%s, not on the one that held it", h.Node, h.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("mesh-store has no holder on record after assigning: %v", holdings)
|
||||
}
|
||||
|
||||
// And the record decides from here: the anchor's plan holds the seat, the laptop's does not.
|
||||
for node, holds := range map[string]bool{"anchor": true, "laptop": false} {
|
||||
plan, _, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatalf("%s no longer resolves: %v", node, err)
|
||||
}
|
||||
var claimed bool
|
||||
for _, c := range plan.Claims {
|
||||
if c.Claim == "mesh-store" {
|
||||
claimed = true
|
||||
}
|
||||
}
|
||||
if claimed != holds {
|
||||
t.Fatalf("%s holds mesh-store: %v, want %v", node, claimed, holds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderOnRecordIsNotRewrittenByDerivation(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := open.inventory.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, aSeatedStore())
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
if _, err := assign(ctx, open, node, "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// A person hands the seat to the laptop. From here the record decides, and what the mesh
|
||||
// derives must never write over it.
|
||||
if err := open.inventory.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "laptop", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := recordDerivedHolders(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 0 {
|
||||
t.Fatalf("a seat on record was written again from derivation: %v", said)
|
||||
}
|
||||
holdings, _ := open.inventory.Holdings(ctx)
|
||||
for _, h := range holdings {
|
||||
if h.Claim == "mesh-store" && h.Node != "laptop" {
|
||||
t.Fatalf("the record moved to %s", h.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A host refuses a declaration carrying a field it does not know, and refuses it whole — so every new
|
||||
// field is a flag day, and the mesh had no record of which host any machine ran (novox/hq
|
||||
// 04-ISSUES/087). The order was kept by somebody remembering it.
|
||||
|
||||
func TestTheMeshNamesWhichMachinesRunWhichHost(t *testing.T) {
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "04a27ca"},
|
||||
{Name: "laptop", HostVersion: "ced54d4"},
|
||||
{Name: "spare", HostVersion: "04a27ca"},
|
||||
})
|
||||
if len(split) != 2 {
|
||||
t.Fatalf("two versions were reported and the split has %d: %v", len(split), split)
|
||||
}
|
||||
if got := strings.Join(split["04a27ca"], ","); got != "anchor,spare" && got != "spare,anchor" {
|
||||
t.Fatalf("04a27ca is held by %q", got)
|
||||
}
|
||||
if got := strings.Join(split["ced54d4"], ","); got != "laptop" {
|
||||
t.Fatalf("ced54d4 is held by %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheMeshDoesNotClaimWhichHostIsNewer(t *testing.T) {
|
||||
// **The fault this replaced.** A host reports its version as a commit, and commits have no order.
|
||||
// The first version compared them as strings and, on the live mesh, named the three machines
|
||||
// running the NEWER host as the ones behind: `ced54d4` sorts above `04a27ca` and means nothing.
|
||||
//
|
||||
// There is no assertion to make about which is newer, and that is the point — the type says so.
|
||||
// hostSplit returns who runs what, and nothing that could be read as an ordering.
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "old-but-sorts-high", HostVersion: "ced54d4"},
|
||||
{Name: "new-but-sorts-low", HostVersion: "04a27ca"},
|
||||
})
|
||||
for version, machines := range split {
|
||||
if len(machines) != 1 {
|
||||
t.Fatalf("%s is held by %v", version, machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineThatHasNotSaidIsNotAVersion(t *testing.T) {
|
||||
// It may be running anything. Counting it as a version would invent a disagreement; `node show`
|
||||
// says per machine that it has not said.
|
||||
split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "04a27ca"},
|
||||
{Name: "quiet"},
|
||||
})
|
||||
if split != nil {
|
||||
t.Fatalf("one reported version and one silence read as a disagreement: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMachinesAgreeingOnTheirHostAreNotADisagreement(t *testing.T) {
|
||||
if split := hostSplit([]inventory.Node{
|
||||
{Name: "anchor", HostVersion: "v2"},
|
||||
{Name: "laptop", HostVersion: "v2"},
|
||||
}); split != nil {
|
||||
t.Fatalf("machines agreeing reported a split: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) {
|
||||
if split := hostSplit([]inventory.Node{{Name: "anchor"}, {Name: "laptop"}}); split != nil {
|
||||
t.Fatalf("a mesh told no host version reported a split: %v", split)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) {
|
||||
// The machine has sent this since ADR 0141 and the controller's own copy of the report did not
|
||||
// have the field, so it was unmarshalled into nothing. End to end through the store, because the
|
||||
// fault was a field that existed on one side of the wire only.
|
||||
open := aMesh(t)
|
||||
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.HostVersion != "" {
|
||||
t.Fatalf("a machine that never reported one has host version %q", record.HostVersion)
|
||||
}
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again.HostVersion != "ced54d4" {
|
||||
t.Fatalf("the reported host version read back as %q", again.HostVersion)
|
||||
}
|
||||
// An empty report never clears what a machine last said: a bare word that the node is there says
|
||||
// nothing about its host.
|
||||
if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.HostVersion != "ced54d4" {
|
||||
t.Fatalf("a report carrying no host version cleared what the machine had said: %q",
|
||||
kept.HostVersion)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (int64, error) {
|
||||
if node == "anchor" {
|
||||
return 0, context.DeadlineExceeded
|
||||
}
|
||||
return 7, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil)
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// licenceCommand is everything about model access the mesh holds.
|
||||
//
|
||||
// **A licence is a named thing and the name is the operator's** (novox/hq ADR 0024). *The personal
|
||||
// account*, *the organisation's account* — those are names a person uses, and the mesh has to use
|
||||
// them too, because the whole point is saying which one a given consumer uses.
|
||||
func licenceCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(
|
||||
"licence add|list|use|release|key|manager|set-grant|refresh|submit-refresh|forget")
|
||||
}
|
||||
switch args[0] {
|
||||
case "add":
|
||||
return licenceAdd(ctx, args[1:])
|
||||
case "list":
|
||||
return licenceList(ctx)
|
||||
case "use":
|
||||
return licenceUse(ctx, args[1:], true)
|
||||
case "release":
|
||||
return licenceUse(ctx, args[1:], false)
|
||||
case "key":
|
||||
return licenceKey(ctx, args[1:])
|
||||
case "manager":
|
||||
return licenceManager(ctx, args[1:])
|
||||
case "set-grant":
|
||||
return licenceSetGrant(ctx, args[1:])
|
||||
case "refresh":
|
||||
return licenceRefresh(ctx, args[1:])
|
||||
case "submit-refresh":
|
||||
return licenceSubmitRefresh(ctx, args[1:])
|
||||
case "forget":
|
||||
return licenceForget(ctx, args[1:])
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"licence %q; it is add, list, use, release, key, manager, set-grant, refresh, "+
|
||||
"submit-refresh or forget", args[0])
|
||||
}
|
||||
|
||||
func licenceAdd(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("licence add", flag.ContinueOnError)
|
||||
// What a consumer must know that is not secret — a base URL, a model name. Never the key.
|
||||
serves := set.String("serves", "",
|
||||
"JSON a consumer must know that is not secret, such as a base URL or a model")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 2 {
|
||||
return errors.New(`licence add <vendor> <name> [--serves '{"model":"..."}']`)
|
||||
}
|
||||
vendor, name := positionals[0], positionals[1]
|
||||
|
||||
values := map[string]any{}
|
||||
if strings.TrimSpace(*serves) != "" {
|
||||
if err := json.Unmarshal([]byte(*serves), &values); err != nil {
|
||||
return fmt.Errorf("--serves is not JSON: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
if err := held.Add(ctx, name, vendor, values); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s) recorded. Nothing uses it yet, and it has no key:\n"+
|
||||
" licence use %s <node> <module>\n licence key %s\n", name, vendor, name, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func licenceList(ctx context.Context) error {
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
|
||||
all, err := held.All(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(all) == 0 {
|
||||
// Said, not printed as nothing: an empty list and a failed read must never look the same.
|
||||
fmt.Println("this mesh holds no licences")
|
||||
return nil
|
||||
}
|
||||
for _, one := range all {
|
||||
holders, err := held.HoldersOf(ctx, one.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s (%s)\n", one.Name, one.Vendor)
|
||||
if len(holders) == 0 {
|
||||
fmt.Printf(" nobody uses it\n")
|
||||
}
|
||||
for _, h := range holders {
|
||||
// Whether it has a key is the question somebody is actually asking, so it is said
|
||||
// per holder rather than per licence: the key was sealed to the holders that existed
|
||||
// when it was supplied, and one recorded afterwards has none.
|
||||
state := "has no key — supply it again with `licence key " + one.Name + "`"
|
||||
if h.Sealed != "" {
|
||||
state = "has a key"
|
||||
}
|
||||
fmt.Printf(" %s on %s: %s\n", h.Module, h.Node, state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func licenceUse(ctx context.Context, args []string, using bool) error {
|
||||
verb := "use"
|
||||
if !using {
|
||||
verb = "release"
|
||||
}
|
||||
if len(args) != 3 {
|
||||
return fmt.Errorf("licence %s <name> <node> <module>", verb)
|
||||
}
|
||||
name, node, module := args[0], args[1], args[2]
|
||||
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
|
||||
if !using {
|
||||
if err := held.StopUsing(ctx, name, node, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s no longer uses %s. Its copy of the key goes on the next push\n",
|
||||
module, node, name)
|
||||
return nil
|
||||
}
|
||||
if err := held.Use(ctx, name, node, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s uses %s.\n", module, node, name)
|
||||
// The consequence, said now rather than discovered as a machine that resolves and receives
|
||||
// nothing: the mesh discarded the plaintext, so a holder added after the key was supplied has
|
||||
// no key and the mesh cannot make one.
|
||||
sealed, err := held.KeyFor(ctx, name, node, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sealed == "" {
|
||||
fmt.Printf(" It has no key yet — the mesh discarded the plaintext when it was supplied "+
|
||||
"and cannot seal another. Supply it again:\n licence key %s\n", name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// licenceKey is the *accept* verb novox/hq ADR 0024 names as missing.
|
||||
//
|
||||
// Take a value, seal it to each holder, and discard the plaintext. Every other credential the
|
||||
// mesh handles it generated itself; an API key arrives from a person, and a mesh that kept
|
||||
// operator-supplied keys readably is the arrangement this project measured and rejected.
|
||||
func licenceKey(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("licence key", flag.ContinueOnError)
|
||||
// A file rather than an argument, by default. A key on a command line is a key in shell
|
||||
// history and in every process listing taken while it ran.
|
||||
from := set.String("file", "", "read the key from a file instead of standard input")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("licence key <name> [--file <path>]")
|
||||
}
|
||||
name := positionals[0]
|
||||
|
||||
var value string
|
||||
if *from != "" {
|
||||
raw, err := os.ReadFile(*from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value = strings.TrimSpace(string(raw))
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "reading the key from standard input; it is not echoed anywhere")
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
line, err := reader.ReadString('\n')
|
||||
if err != nil && line == "" {
|
||||
return fmt.Errorf("nothing was given on standard input: %w", err)
|
||||
}
|
||||
value = strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
sealed, err := held.Accept(ctx, name, value, func(node string) (string, error) {
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
})
|
||||
if err != nil {
|
||||
if sealed > 0 {
|
||||
// Some holders got it and some did not, and the person holding the key is the only
|
||||
// one who can finish the job. Saying how far it got is the difference between running
|
||||
// this again knowing what it will do and running it hoping.
|
||||
return fmt.Errorf(
|
||||
"%w\n\n%d holder(s) were sealed before this. Running `licence key %s` again "+
|
||||
"with the same key seals the rest and changes nothing for those already done",
|
||||
err, sealed, name)
|
||||
}
|
||||
return err
|
||||
}
|
||||
// Not echoed back, ever. What is stored is unreadable by whoever holds it, the mesh included,
|
||||
// and printing the value here would put the one copy that matters on a terminal.
|
||||
fmt.Printf("sealed to %d holder(s). The mesh has discarded the key and cannot read it back\n",
|
||||
sealed)
|
||||
fmt.Printf(" run `push` to deliver it\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// licenceManager names the one node that holds a refreshable-grant licence's refresh token readably
|
||||
// and refreshes it centrally (novox/hq ADR 0050).
|
||||
//
|
||||
// **Only a refreshable-grant licence has one.** A static-key licence has no refresh token to hold, so
|
||||
// naming a manager for it is refused where the mistake is made rather than kept as a field that means
|
||||
// nothing — the absent manager is part of what keeps a static key from ever holding a value readably
|
||||
// at rest.
|
||||
func licenceManager(ctx context.Context, args []string) error {
|
||||
if len(args) != 3 {
|
||||
return errors.New("licence manager <name> <node> <module>")
|
||||
}
|
||||
name, node, module := args[0], args[1], args[2]
|
||||
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
if err := held.SetManager(ctx, name, node, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s holds and refreshes %s.\n"+
|
||||
" Its refresh token is sealed to %s's key — readable by that node alone, not by any other "+
|
||||
"node and not by this database. Put %s on the licence too so it is delivered the token:\n"+
|
||||
" licence use %s %s %s\n", module, node, name, node, module, name, node, module)
|
||||
return nil
|
||||
}
|
||||
|
||||
// sealedGrantJSON is the wire shape of a sealed refresh token on this command surface: an anonymous
|
||||
// sealed box and the public key it was sealed to, and nothing else.
|
||||
//
|
||||
// **Every field of it is ciphertext or a public key.** `sealed` is the refresh token as a
|
||||
// `crypto_box_seal` to the manager node's public key; `manager_key` is that public key. Neither is
|
||||
// the refresh token in the clear — which is why this surface may read one in (`set-grant`,
|
||||
// `submit-refresh`) without the control plane ever holding a refresh token it could read. The manager
|
||||
// module, on the manager node, seals it; the HOST, on that node, unseals it to deliver cleartext. This
|
||||
// database, and this surface, only ever forward the box (novox/hq ADR 0050).
|
||||
type sealedGrantJSON struct {
|
||||
Sealed string `json:"sealed"`
|
||||
ManagerKey string `json:"manager_key"`
|
||||
}
|
||||
|
||||
// readSealedGrant reads a sealed refresh token from a file or standard input as JSON.
|
||||
func readSealedGrant(from string) (sealedGrantJSON, error) {
|
||||
var raw []byte
|
||||
var err error
|
||||
if from != "" {
|
||||
raw, err = os.ReadFile(from)
|
||||
} else {
|
||||
raw, err = readAllStdin()
|
||||
}
|
||||
if err != nil {
|
||||
return sealedGrantJSON{}, err
|
||||
}
|
||||
var g sealedGrantJSON
|
||||
if err := json.Unmarshal(raw, &g); err != nil {
|
||||
return sealedGrantJSON{}, fmt.Errorf("the sealed refresh token is not JSON: %w", err)
|
||||
}
|
||||
if g.Sealed == "" || g.ManagerKey == "" {
|
||||
return sealedGrantJSON{}, errors.New(
|
||||
"a sealed refresh token is {sealed, manager_key}, and one part is missing")
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
func readAllStdin() ([]byte, error) {
|
||||
reader := bufio.NewReader(os.Stdin)
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
// licenceSetGrant stores a sealed refresh token the manager module produced — adoption, and the
|
||||
// re-seal after a rotation done outside this process (novox/hq ADR 0050).
|
||||
//
|
||||
// **It takes a sealed box, never a refresh token.** The manager module, on the manager node, reads
|
||||
// the operator's refresh token, seals it to that node's own public key, and hands the box here. So the
|
||||
// one moment a refresh token is in the clear is on the manager node, never in the control plane — the
|
||||
// same bound the whole carve-out keeps. This surface refuses anything that is not a complete sealed
|
||||
// grant rather than storing half of one.
|
||||
func licenceSetGrant(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("licence set-grant", flag.ContinueOnError)
|
||||
from := set.String("file", "", "read the sealed refresh token from a file instead of standard input")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("licence set-grant <name> [--file <path>]")
|
||||
}
|
||||
name := positionals[0]
|
||||
|
||||
grant, err := readSealedGrant(*from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
if err := held.SetRefreshGrant(ctx, name, grant.Sealed, grant.ManagerKey); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s now holds a refresh token for %s, sealed to that node's key and readable by it "+
|
||||
"alone.\n the control plane stored the box without opening it; run `push` to deliver it\n",
|
||||
"the manager", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// licenceSubmitRefresh publishes a refresh a MANAGER NODE already performed: the new access token is
|
||||
// sealed to every holder, and a rotated refresh token replaces the stored envelope (novox/hq ADR
|
||||
// 0050, Phase C).
|
||||
//
|
||||
// **This is the boundary the invariant rests on.** The manager runtime, on the manager node, opened
|
||||
// the at-rest envelope with that node's key, called the vendor's OAuth endpoint, and produced this:
|
||||
// the new access token in the clear, and — only if the vendor rotated it — the refresh token already
|
||||
// re-sealed at rest. This reads exactly those two things and no refresh token in the clear ever
|
||||
// reaches it, because it is never given one. The access token is sealed per holder and discarded,
|
||||
// as any accepted key is; the rotated envelope is stored opaque.
|
||||
func licenceSubmitRefresh(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("licence submit-refresh", flag.ContinueOnError)
|
||||
accessFrom := set.String("access-file", "",
|
||||
"read the new access token from a file instead of standard input")
|
||||
grantFrom := set.String("grant-file", "",
|
||||
"the rotated sealed refresh token, if the vendor rotated it; omit if it did not")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New(
|
||||
"licence submit-refresh <name> [--access-file <path>] [--grant-file <path>]")
|
||||
}
|
||||
name := positionals[0]
|
||||
|
||||
var accessToken string
|
||||
if *accessFrom != "" {
|
||||
raw, err := os.ReadFile(*accessFrom)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accessToken = strings.TrimSpace(string(raw))
|
||||
} else {
|
||||
raw, err := readAllStdin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
accessToken = strings.TrimSpace(string(raw))
|
||||
}
|
||||
if accessToken == "" {
|
||||
return errors.New("no access token was given, so there is nothing to seal")
|
||||
}
|
||||
|
||||
// The rotated sealed token is optional: absent, the stored refresh token is left exactly as it was.
|
||||
var newSealed, newManagerKey string
|
||||
if *grantFrom != "" {
|
||||
grant, err := readSealedGrant(*grantFrom)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newSealed, newManagerKey = grant.Sealed, grant.ManagerKey
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.Licences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inv := open.inventory
|
||||
|
||||
sealed, err := held.SubmitRefresh(ctx, name, accessToken, newSealed, newManagerKey,
|
||||
func(node string) (string, error) {
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rotatedNote := "the refresh token was left with its manager unchanged"
|
||||
if newSealed != "" {
|
||||
rotatedNote = "the rotated refresh token replaced the stored box, still readable by the " +
|
||||
"manager node alone"
|
||||
}
|
||||
fmt.Printf("submitted a refresh for %s: a new access token sealed to %d holder(s), and %s.\n"+
|
||||
" run `push` to deliver it\n", name, sealed, rotatedNote)
|
||||
return nil
|
||||
}
|
||||
|
||||
// licenceRefresh mints a new access token for a refreshable-grant licence and seals it to every
|
||||
// holder (novox/hq ADR 0050). The refresh token stays with the manager and is never delivered.
|
||||
//
|
||||
// The vendor's actual refresh is a plug-in this build does not ship (Phase C), so here this reports
|
||||
// that plainly rather than pretending to have refreshed.
|
||||
func licenceRefresh(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("licence refresh <name>")
|
||||
}
|
||||
name := args[0]
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
held, err := open.Licences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inv := open.inventory
|
||||
|
||||
sealed, err := held.Refresh(ctx, name, func(node string) (string, error) {
|
||||
return inv.SealingKeyOf(ctx, node)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("refreshed %s: a new access token sealed to %d holder(s), and the refresh token left "+
|
||||
"with its manager.\n run `push` to deliver it\n", name, sealed)
|
||||
return nil
|
||||
}
|
||||
|
||||
func licenceForget(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("licence forget <name>")
|
||||
}
|
||||
held, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer held.Close()
|
||||
if err := held.Forget(ctx, args[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
// Said plainly, because the mesh cannot do it and pretending otherwise is worse than useless:
|
||||
// a licence outliving its holder is a live credential nobody is watching.
|
||||
fmt.Printf("%s is forgotten, and every record of who held it with it.\n"+
|
||||
" The key itself is not the mesh's to revoke — do that where the licence was bought\n",
|
||||
args[0])
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
// Command mesh-controller is the control plane: everything that needs to know about more than one
|
||||
// node (novox/hq ADR 0006).
|
||||
//
|
||||
// It runs as one process holding several contexts, each owning its own store. Today it holds one,
|
||||
// `inventory`, and does one thing with it — brings its schema up to date, which is step 3 of the
|
||||
// bootstrap in novox/hq 07-the-foundation and the step the first node cannot get past without.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
// version is stamped at link time. Unset in a development build, and it says so rather than
|
||||
// claiming a number.
|
||||
var version = "development build"
|
||||
|
||||
// held is a context this process was granted, and the schema it carries.
|
||||
//
|
||||
// novox/hq ADR 0006 names seven. One is built. The list is short because the others do not exist
|
||||
// yet, not because they are optional.
|
||||
var held = []struct {
|
||||
name string
|
||||
migrations func() ([]store.Migration, error)
|
||||
}{
|
||||
{inventory.Name, inventory.Migrations},
|
||||
{identity.Name, identity.Migrations},
|
||||
{licences.Name, licences.Migrations},
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
args := os.Args[1:]
|
||||
if len(args) == 0 {
|
||||
usage()
|
||||
return fmt.Errorf("no command given")
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
return buildCommand(ctx, args[1:])
|
||||
case "builder":
|
||||
return builderCommand(ctx, args[1:])
|
||||
case "board":
|
||||
return boardCommand(ctx, args[1:])
|
||||
case "api":
|
||||
return apiCommand(ctx, args[1:])
|
||||
case "licence":
|
||||
return licenceCommand(ctx, args[1:])
|
||||
case "rotate":
|
||||
return rotateCommand(ctx, args[1:])
|
||||
case "ask":
|
||||
return askCommand(ctx, args[1:])
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return queueCommand(ctx, args[1:])
|
||||
case "cancel":
|
||||
return cancelCommand(ctx, args[1:])
|
||||
case "clear":
|
||||
return clearCommand(ctx, args[1:])
|
||||
case "rebuild":
|
||||
return rebuildCommand(ctx, args[1:])
|
||||
case "replay":
|
||||
return replayCommand(ctx, args[1:])
|
||||
case "kill":
|
||||
return killCommand(ctx, args[1:])
|
||||
case "pause", "resume":
|
||||
return pauseCommand(ctx, args[0], args[1:])
|
||||
case "collection":
|
||||
return collectionCommand(ctx, args[1:])
|
||||
case "plans":
|
||||
return plansCommand(ctx, args[1:])
|
||||
case "pin":
|
||||
return pinCommand(ctx, args[1:], true)
|
||||
case "unpin":
|
||||
return pinCommand(ctx, args[1:], false)
|
||||
// `prepare` is how the mesh asks any module to bring its state to the shape this version needs
|
||||
// (novox/hq ADR 0135), and the control plane answers it the same way as everything else — its
|
||||
// own schema is not a special case. `migrate` remains the word a person types.
|
||||
case "prepare", "migrate":
|
||||
return migrate(ctx)
|
||||
case "node":
|
||||
return nodeCommand(ctx, args[1:])
|
||||
case "token":
|
||||
return tokenCommand(ctx, args[1:])
|
||||
case "identity":
|
||||
return identityCommand(ctx, args[1:])
|
||||
case "broker":
|
||||
return brokerCommand(ctx, args[1:])
|
||||
case "serve":
|
||||
return serve(ctx)
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
case "declare":
|
||||
return declare(ctx, args[1:])
|
||||
case "overlay":
|
||||
return overlayCommand(ctx, args[1:])
|
||||
case "module":
|
||||
return moduleCommand(ctx, args[1:])
|
||||
case "assign", "unassign":
|
||||
return assignCommand(ctx, args[0], args[1:])
|
||||
case "take":
|
||||
return takeCommand(ctx, args[1:])
|
||||
case "converge":
|
||||
return convergeCommand(ctx, args[1:])
|
||||
case "adopt":
|
||||
return adoptCommand(ctx, args[1:])
|
||||
case "settings":
|
||||
return settingsCommand(ctx, args[1:])
|
||||
case "secret":
|
||||
return secretCommand(ctx, args[1:])
|
||||
case "operator":
|
||||
return operatorCommand(ctx, args[1:])
|
||||
case "plan":
|
||||
return planCommand(ctx, args[1:])
|
||||
case "push":
|
||||
return pushCommand(ctx, args[1:])
|
||||
case "rollout":
|
||||
return rolloutCommand(ctx, args[1:])
|
||||
case "seats":
|
||||
return seatsCommand(ctx, args[1:])
|
||||
case "seat":
|
||||
return seatCommand(ctx, args[1:])
|
||||
case "status":
|
||||
return statusCommand(ctx, args[1:])
|
||||
case "version":
|
||||
fmt.Println(version)
|
||||
return nil
|
||||
case "help", "-h", "--help":
|
||||
usage()
|
||||
return nil
|
||||
default:
|
||||
usage()
|
||||
return fmt.Errorf("%q is not a command", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprint(os.Stderr, `mesh-controller — the control plane
|
||||
|
||||
migrate bring each context's schema up to date
|
||||
prepare the same, asked the way the mesh asks any module (ADR 0135)
|
||||
node add <name> [--adopted] create a node record; --adopted: the machine is in use
|
||||
node list the nodes this mesh knows about
|
||||
node show <name> what one machine reported it can do, and why
|
||||
node public-domain <name> the domain it composes its routed names under
|
||||
node public-domain <name> <d> ...set it to d
|
||||
node public-domain <name> --clear ...it faces the outside no longer
|
||||
node networks <name> the networks it routes for what it hosts
|
||||
node networks <name> <cidr>... ...set them; its filter forwards these too
|
||||
node networks <name> --clear ...only the container runtime's own
|
||||
token issue --node <name> a one-time right to join, for an existing record
|
||||
token issue --new <name> create the record and issue for it
|
||||
token issue ... --adopted ...for a machine in use, which joins adopted
|
||||
identity show this control plane's signing key
|
||||
broker show where the broker is, and what to expect there
|
||||
serve consume what nodes say, and answer
|
||||
declare <node> <file> send a node a signed declaration
|
||||
overlay place <node> [flags] say where a node is and how it is reached
|
||||
overlay show the private network, as the mesh computes it
|
||||
module add <file> register a module from its manifest
|
||||
module check <file|dir>... judge manifests where they are written, with no mesh (exit 1 on any problem)
|
||||
module list what modules this mesh knows about
|
||||
module moved <name> <commit> the source has a newer commit than the mesh built
|
||||
module forget <name> remove one, unless a node runs it or the mesh holds things for it
|
||||
module forget <name> --and-what-it-holds ...and discard its settings, secrets and ports too
|
||||
module issue <name> --node <m> a broker account for a module, scoped to its emits and consumes
|
||||
upgrade <name> what happens when this module's current version moves
|
||||
upgrade <name> roll-out [--together] ...send it to the machines running it
|
||||
upgrade <name> record ...record that they are behind, and send nothing
|
||||
status [--json] what is wrong, what is quiet, and what is out of date
|
||||
seats [--json] every seat this mesh defines, what it delivers, and who holds it
|
||||
seat rename <from> <to> rename a seat; its former name still resolves (ADR 0122)
|
||||
seat <name> --to <node>/<module> hand a seat to that assignment as one act; never empty in between (ADR 0131)
|
||||
seat <name> --add <node>/<module> add a holder beside the others, for a replicated seat (ADR 0223)
|
||||
board [--listen ADDR] the same three questions, as a page that holds nothing
|
||||
api --issuer URL [--listen A] assign and unassign over http, for a surface that is not here
|
||||
assign <node> <module>... put modules on a node, judged together (ADR 0207)
|
||||
unassign <node> <module>... take them off
|
||||
take <node> <module> preview a module's cutover on an adopted node: what runs beside
|
||||
what it declares; --yes <digest> cuts it over as previewed
|
||||
converge <node> [--yes <digest>] [--filter nftables] preview, then make, an adopted node converged
|
||||
adopt <node> return a converged node to adopted; what was taken stays taken
|
||||
settings set <module> <file> what a module's config should say, for the whole mesh
|
||||
settings set <module> <file> --node <n> ...or for one machine
|
||||
settings clear <module> [--node <n>] take a layer away
|
||||
secret accept <node> <module> <name> carry a value the mesh did not make and cannot invent
|
||||
secret accept ... --from <file> ...read it from a file rather than being asked
|
||||
secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>]
|
||||
break-glass: open the operator-sealed copy, to a 0600 file
|
||||
secret export [--out <file>] every operator-sealed copy, ciphertext — keep it with the key
|
||||
operator key make [--out <file>] make the operator's sealing key, off the mesh; private half to the file only
|
||||
operator key set <public> [--replace] tell the mesh which operator key to seal to
|
||||
operator key show the operator key, and what it can recover
|
||||
build <repository> [--ref R] have a build machine build it, and record what came out
|
||||
build --behind build every module the mesh holds older than its source
|
||||
build --on <module> rebuild every module that stands on this module's artifacts, bases first
|
||||
builds [<module>] what has been built lately, and what came of it
|
||||
queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead
|
||||
cancel <id> drop a waiting or dead ask; recorded failed, cancelled by hand
|
||||
clear [--dead] cancel every waiting ask (and the dead ones); never one in flight
|
||||
rebuild <module|build-id> ask the module's source again, or that build's, under a new id
|
||||
replay <build-id> [--register [--older]] that build's commit again; a dry run unless --register
|
||||
kill <id> end a build where it runs; recorded failed, killed by hand
|
||||
pause [<node>] / resume [<node>] the build seat's holder there, or every holder, takes nothing new / again
|
||||
plans retry <id> ask a failed plan's failed builds again, and carry the plan on
|
||||
collection [--json] kept archives held/unheld by a manifest, and what the sweep may let go
|
||||
builder issue <name> a broker account for a build machine, scoped to build work,
|
||||
delivered as the builder module's broker secret (module add it first)
|
||||
licence add|list|use|key model access, under the name a person calls it
|
||||
licence manager <name> <node> the node that holds a refreshable licence's refresh token
|
||||
licence refresh <name> mint a new access token and seal it to every holder
|
||||
rotate <provision> [--consumer <n>] a new credential for every holder, both ends at once
|
||||
ask <module> <tool> [json] call one of a module's tools over the broker, and print its answer
|
||||
pin <node> <provision> <from-node> <module>
|
||||
which provider this one gets a provision from: the module, and its node
|
||||
unpin <node> <provision> put that question back
|
||||
plan <node> [--files|--json] what that node would run, and why
|
||||
push [<node>] [--behind] send a node everything it should be, or only those that need it
|
||||
version what this binary is
|
||||
|
||||
Each context reaches its own store through its own credential (novox/hq ADR 0008), named
|
||||
`+store.Variable("<context>")+` — or `+store.FileVariable("<context>")+`, naming a file that holds
|
||||
the same thing and keeps the password out of the environment. This process holds:
|
||||
|
||||
`)
|
||||
for _, c := range held {
|
||||
fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n",
|
||||
c.name, store.Database(c.name), store.Variable(c.name))
|
||||
}
|
||||
fmt.Fprintln(os.Stderr)
|
||||
}
|
||||
|
||||
// parseAround reads flags that may sit before, after or between positional arguments.
|
||||
//
|
||||
// The standard library stops at the first non-flag argument, so `module add thing.json --source x`
|
||||
// parses no flags at all and silently ignores every one of them. The host learned this the same
|
||||
// way and says so in its own parser: a flag that is quietly dropped is the fault this project
|
||||
// keeps naming, and it looks exactly like success.
|
||||
func parseAround(set *flag.FlagSet, args []string) ([]string, error) {
|
||||
var positionals []string
|
||||
rest := args
|
||||
for {
|
||||
if err := set.Parse(rest); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rest = set.Args()
|
||||
if len(rest) == 0 {
|
||||
return positionals, nil
|
||||
}
|
||||
positionals = append(positionals, rest[0])
|
||||
rest = rest[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// Built is the daemon hearing a build's outcome on the bus — its own asking, an announcement's, or
|
||||
// a tool's that did not wait (novox/hq issue 176) — and taking it in: recorded, and the module
|
||||
// registered, the same as the waiting command does. Said either way, so the daemon's log tells what
|
||||
// became of a build nobody was watching.
|
||||
func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
||||
// **A dry run is looked at, never taken in** (novox/hq issue 240). On 2026-10-04 a dry run of an
|
||||
// unmerged branch was heard here like any build, registered, and its definition reached a machine
|
||||
// before anyone had reviewed it.
|
||||
if result.DryRun {
|
||||
fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref)
|
||||
return nil
|
||||
}
|
||||
manifest, _, err := takeIn(ctx, b.inv, result)
|
||||
// When it was asked, so a plan takes as its outcome only a build asked for it or after it
|
||||
// (novox/hq 04-ISSUES/219). Zero when the id does not say.
|
||||
asked, _ := link.BuildAskedAt(result.ID)
|
||||
switch {
|
||||
case err != nil && result.Failed != "":
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
if result.Module != "" {
|
||||
planBuilt(ctx, b.open, result.Module, result.Commit, result.Failed, asked, result.ID)
|
||||
} else {
|
||||
planFailedBuild(ctx, b.open, result)
|
||||
}
|
||||
return nil
|
||||
case errors.Is(err, inventory.ErrSuperseded):
|
||||
// Not a failure: the module is already at what a later request built. A plan that asked
|
||||
// before that later request is answered by it; one that asked after it ignores this.
|
||||
fmt.Printf("%s: %v\n", result.ID, err)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
case err != nil:
|
||||
fmt.Printf("%s: heard and recorded, and not registered: %v\n", result.ID, err)
|
||||
if manifest.Module != "" {
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, err.Error(), asked, result.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s %s registered, built on %s from %s\n",
|
||||
result.ID, manifest.Module, manifest.Version, result.On, short(result.Commit))
|
||||
saysWhenThePolicyActs(ctx, b.inv, manifest.Module)
|
||||
planBuilt(ctx, b.open, manifest.Module, result.Commit, "", asked, result.ID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdh"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// A mesh a command can be run against.
|
||||
//
|
||||
// The commands here were tested through the pieces they call and never through themselves, so
|
||||
// three faults that only exist where the pieces meet — an assignment reported as fine while it
|
||||
// blocked other machines, a read-shaped invocation that wrote, a JSON interface that stopped
|
||||
// emitting JSON — were invisible to every test in this package. This raises the real stores and
|
||||
// calls the real functions.
|
||||
|
||||
// aMesh is two placed, capable machines on a private network, with nothing assigned but the
|
||||
// network itself.
|
||||
//
|
||||
// `anchor` is the hub. That is not decoration: a mesh whose hub cannot be resolved has no private
|
||||
// network at all, which is how one machine's problem reaches every other.
|
||||
func aMesh(t *testing.T) *stores {
|
||||
t.Helper()
|
||||
inventory.ForTest(t) // raises the store, migrates it, and points the environment at it
|
||||
licences.ForTest(t) // planning reaches this one too, by name and never by connection
|
||||
|
||||
open, err := openStores(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(open.Close)
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(t.Context(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for i, name := range []string{"anchor", "laptop"} {
|
||||
record, err := open.inventory.AddNode(t.Context(), name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetPlace(t.Context(), name, name+".example:51820", "here",
|
||||
name == "anchor", fmt.Sprintf("10.77.0.%d", i+1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
||||
{"name": "container-runtime", "present": true},
|
||||
{"name": "wireguard", "present": true},
|
||||
{"name": "systemd", "present": true},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var profile map[string]any
|
||||
if err := json.Unmarshal(reported, &profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordOverlayKey(t.Context(), record.ID, aPublicKey(t)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(t.Context(), name, overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return open
|
||||
}
|
||||
|
||||
// aPublicKey is a key a machine could have reported. Its private half is thrown away: nothing here
|
||||
// opens anything, it only needs the mesh to believe a machine has a key.
|
||||
func aPublicKey(t *testing.T) string {
|
||||
t.Helper()
|
||||
k, err := ecdh.X25519().GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(k.PublicKey().Bytes())
|
||||
}
|
||||
|
||||
// register puts a manifest in the catalogue.
|
||||
func register(t *testing.T, open *stores, m catalogue.Manifest) {
|
||||
t.Helper()
|
||||
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// rivals are two modules that cannot share a machine, which is the shortest way to make a node's
|
||||
// own set of assignments incoherent using nothing but commands a person has.
|
||||
func rivals() (catalogue.Manifest, catalogue.Manifest) {
|
||||
claim := []catalogue.Claim{{Name: "the-seat", Scope: catalogue.ScopeNode}}
|
||||
return catalogue.Manifest{Module: "rival-one", Version: "1", Claims: claim},
|
||||
catalogue.Manifest{Module: "rival-two", Version: "1", Claims: claim}
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge the bus announced and the controller never acted on is caught up** (novox/hq issue 266).
|
||||
//
|
||||
// The forge's poll announces every merge on the events stream, and the controller acts on what its
|
||||
// consumer there hands it. On 2026-10-06 one merge was on the stream and never handed over: the bus
|
||||
// server moved the consumer past it — a fault of consumers with several filters in the server the
|
||||
// mesh ran — and the controller, which only acts on what it is handed, said nothing. The modules
|
||||
// built from that repository stayed behind and were built by hand.
|
||||
//
|
||||
// So the stream is read back on a timer, on a consumer of its own filtered on merges alone, and every
|
||||
// announcement older than mergeGrace is judged as SourceMoved would judge it. **No record of what
|
||||
// was handled is kept, because none is needed**: acting on a merge marks every module it moved as
|
||||
// looked at since, so an announcement already acted on reads as history and moves nothing. One that
|
||||
// would still move something was never acted on — it is said, and acted on now.
|
||||
const (
|
||||
// mergeGrace is how long an announcement is left to the controller's own consumer before it is
|
||||
// judged missed. That consumer hands over one event at a time, and a merge waits behind a build
|
||||
// outcome that is being acted on; acting on a merge itself asks builds and does not wait for them.
|
||||
mergeGrace = 10 * time.Minute
|
||||
// mergeLookBack is how far back a pass reads. A merge missed longer ago than this was missed by a
|
||||
// controller that was not running this, and is the operator's to look at, not a surprise rebuild.
|
||||
mergeLookBack = 24 * time.Hour
|
||||
// mergeCatchUpEvery is how often the stream is read back.
|
||||
mergeCatchUpEvery = 5 * time.Minute
|
||||
)
|
||||
|
||||
// merges is what reads back the forge's announcements; the link server, or a test's list.
|
||||
type merges interface {
|
||||
AnnouncedMerges(ctx context.Context, since time.Time) ([]link.AnnouncedMerge, error)
|
||||
}
|
||||
|
||||
// catchingUpOnMerges reads back the forge's announcements on a timer, until the context ends.
|
||||
func catchingUpOnMerges(ctx context.Context, open *stores, announced merges) {
|
||||
f := following{open}
|
||||
catalogued := func(ctx context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
read, err := open.inventory.ReadRepositories(ctx)
|
||||
return entries, read, err
|
||||
}
|
||||
failing := ""
|
||||
tick := time.NewTicker(mergeCatchUpEvery)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
}
|
||||
err := catchUpOnMerges(ctx, time.Now(), announced, catalogued, f.SourceMoved, func(format string, args ...any) {
|
||||
fmt.Printf(format+"\n", args...)
|
||||
})
|
||||
// A pass that cannot read says so once, not every five minutes, and says when it reads again.
|
||||
why := ""
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != failing {
|
||||
if why != "" {
|
||||
fmt.Printf("merges the bus may not have handed over cannot be looked for: %s\n", why)
|
||||
} else {
|
||||
fmt.Println("merges the bus may not have handed over are looked for again")
|
||||
}
|
||||
failing = why
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// catchUpOnMerges is one pass: every announcement older than mergeGrace that acting on would still
|
||||
// move something is said and acted on, oldest first.
|
||||
//
|
||||
// Judged twice: once against the catalogue as the pass found it, and again just before acting,
|
||||
// because acting on an earlier missed merge of the same repository may have moved what a later one
|
||||
// would have.
|
||||
func catchUpOnMerges(ctx context.Context, now time.Time, announced merges,
|
||||
catalogued func(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error),
|
||||
act func(context.Context, link.SourceMoved) error, say func(string, ...any)) error {
|
||||
|
||||
all, err := announced.AnnouncedMerges(ctx, now.Add(-mergeLookBack))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, read, err := catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range all {
|
||||
if now.Sub(a.At) < mergeGrace {
|
||||
continue
|
||||
}
|
||||
if len(wouldMove(a.SourceMoved, entries, read)) == 0 {
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
moves := wouldMove(a.SourceMoved, entries, read)
|
||||
if len(moves) == 0 {
|
||||
continue
|
||||
}
|
||||
var names []string
|
||||
for _, e := range moves {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
say("%s/%s merged into %s (%.8s), announced %s ago, and the controller never acted on it: the bus "+
|
||||
"did not hand the announcement over (novox/hq issue 266). %s %s behind it; acting on it now",
|
||||
a.Owner, a.Repo, a.Base, a.Commit, now.Sub(a.At).Round(time.Minute), readableList(names),
|
||||
isAre(len(names)))
|
||||
if err := act(ctx, a.SourceMoved); err != nil {
|
||||
say("%s/%s moved to %.8s and the mesh could not act on it: %v; the next pass tries again",
|
||||
a.Owner, a.Repo, a.Commit, err)
|
||||
continue
|
||||
}
|
||||
if entries, read, err = catalogued(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// announcedList is the events stream's merges as a test gives them.
|
||||
type announcedList []link.AnnouncedMerge
|
||||
|
||||
func (a announcedList) AnnouncedMerges(_ context.Context, since time.Time) ([]link.AnnouncedMerge, error) {
|
||||
var out []link.AnnouncedMerge
|
||||
for _, m := range a {
|
||||
if !m.At.Before(since) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// aCatalogue is what the inventory holds, changed the way acting on a merge changes it: every module
|
||||
// the merge moved is marked as looked at (inventory.SourceMoved writes source_seen = now()).
|
||||
type aCatalogue struct {
|
||||
entries []inventory.Entry
|
||||
acted []string
|
||||
fail error
|
||||
}
|
||||
|
||||
func (c *aCatalogue) read(context.Context) ([]inventory.Entry, map[string][]inventory.ReadRepository, error) {
|
||||
return append([]inventory.Entry(nil), c.entries...), nil, nil
|
||||
}
|
||||
|
||||
func (c *aCatalogue) act(now func() time.Time) func(context.Context, link.SourceMoved) error {
|
||||
return func(_ context.Context, m link.SourceMoved) error {
|
||||
if c.fail != nil {
|
||||
return c.fail
|
||||
}
|
||||
c.acted = append(c.acted, m.Repo+"@"+m.Commit[:8])
|
||||
for _, moved := range wouldMove(m, c.entries, nil) {
|
||||
for i := range c.entries {
|
||||
if c.entries[i].Manifest.Module == moved.Manifest.Module {
|
||||
c.entries[i].Source.Head = m.Commit
|
||||
c.entries[i].Source.Seen = now()
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func at(s string) time.Time {
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func announced(repo, commit, mergedAt, onTheBus string, paths ...string) link.AnnouncedMerge {
|
||||
return link.AnnouncedMerge{
|
||||
SourceMoved: link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: commit,
|
||||
MergedAt: mergedAt, Paths: paths,
|
||||
CloneURL: "http://forge.internal:20000/novox/" + repo + ".git"},
|
||||
At: at(onTheBus),
|
||||
}
|
||||
}
|
||||
|
||||
func built(module, repo, path, commit, seen string) inventory.Entry {
|
||||
e := fromRepo(module, "http://forge.internal:20000/novox/"+repo+".git", path)
|
||||
e.Source.BuiltFrom, e.Source.Head, e.Source.Seen = commit, commit, at(seen)
|
||||
return e
|
||||
}
|
||||
|
||||
// **novox/hq issue 266, as it happened.** The forge announced a merge of the tools repository on the
|
||||
// events stream; the bus never handed it to the controller, which acted on the merges around it and
|
||||
// not on this one, and said nothing. Read back from the stream, it is the one merge that would still
|
||||
// move something — so it is said and acted on, once, and only after the controller's own consumer
|
||||
// has had its time with it.
|
||||
func TestAMergeTheBusNeverHandedOverIsActedOnLate(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{
|
||||
built("mesh-tools", "mesh-tools", "", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
built("node-tools", "mesh-tools", "node-tools", "8b789578aaaaaaaa", "2026-10-04T15:24:32Z"),
|
||||
// Acted on when it was announced: looked at after it was merged.
|
||||
built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T22:39:21Z"),
|
||||
}}
|
||||
stream := announcedList{
|
||||
// Nothing the mesh holds is built from the records repository.
|
||||
announced("hq", "88f7f79fcccccccc", "2026-10-05T22:43:00Z", "2026-10-05T22:43:04Z", "04-ISSUES/x.md"),
|
||||
// Acted on: its module was looked at since.
|
||||
announced("mesh-catalog", "78328d4adddddddd", "2026-10-05T22:39:00Z", "2026-10-05T22:39:21Z", "modules/gitea/x.ts"),
|
||||
// Never handed over.
|
||||
announced("mesh-tools", "9730bd89c3e48d0e", "2026-10-05T22:46:47Z", "2026-10-05T22:47:06Z",
|
||||
"node-tools/internal/console/console.go"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:50:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
pass := func() {
|
||||
t.Helper()
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
pass()
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge three minutes old was taken from the controller's own consumer: %v", cat.acted)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T22:58:00Z")
|
||||
pass()
|
||||
if strings.Join(cat.acted, ",") != "mesh-tools@9730bd89" {
|
||||
t.Fatalf("acted on %v, wanted the one merge never handed over", cat.acted)
|
||||
}
|
||||
if len(said) != 1 || !strings.Contains(said[0], "novox/mesh-tools merged into main (9730bd89)") ||
|
||||
!strings.Contains(said[0], "mesh-tools and node-tools are behind it") {
|
||||
t.Fatalf("the missed merge was not said as one: %q", said)
|
||||
}
|
||||
|
||||
clock = at("2026-10-05T23:03:00Z")
|
||||
pass()
|
||||
if len(cat.acted) != 1 || len(said) != 1 {
|
||||
t.Fatalf("a merge acted on was acted on again: %v %q", cat.acted, said)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge that changed none of the held modules' files moves nothing, so it is never "missed"; one
|
||||
// that could not be acted on is said and tried again on the next pass.
|
||||
func TestAMissedMergeThatCouldNotBeActedOnIsTriedAgain(t *testing.T) {
|
||||
cat := &aCatalogue{
|
||||
entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-05T20:00:00Z")},
|
||||
fail: errors.New("the store is restarting"),
|
||||
}
|
||||
stream := announcedList{
|
||||
announced("mesh-catalog", "aaaaaaaa11111111", "2026-10-05T21:00:00Z", "2026-10-05T21:00:10Z",
|
||||
"modules/plex/module.json", "modules/plex/x.ts"),
|
||||
announced("mesh-catalog", "bbbbbbbb22222222", "2026-10-05T21:10:00Z", "2026-10-05T21:10:10Z", "modules/gitea/x.ts"),
|
||||
}
|
||||
var said []string
|
||||
say := func(format string, args ...any) { said = append(said, fmt.Sprintf(format, args...)) }
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
now := func() time.Time { return clock }
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(said) != 2 || !strings.Contains(said[1], "could not act on it") {
|
||||
t.Fatalf("a failed catch-up was not said: %q", said)
|
||||
}
|
||||
cat.fail = nil
|
||||
clock = at("2026-10-05T22:05:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(now), say); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(cat.acted, ",") != "mesh-catalog@bbbbbbbb" {
|
||||
t.Fatalf("acted on %v, wanted only the merge that changed a held module", cat.acted)
|
||||
}
|
||||
}
|
||||
|
||||
// A merge older than the look-back is left to the operator: a controller that did not run this
|
||||
// missed it, and acting on it days later would be a surprise rebuild.
|
||||
func TestAMergeOlderThanTheLookBackIsLeftAlone(t *testing.T) {
|
||||
cat := &aCatalogue{entries: []inventory.Entry{built("gitea", "mesh-catalog", "modules/gitea", "5c2157b8bbbbbbbb", "2026-10-01T00:00:00Z")}}
|
||||
stream := announcedList{announced("mesh-catalog", "cccccccc33333333", "2026-10-03T00:00:00Z", "2026-10-03T00:00:05Z", "modules/gitea/x.ts")}
|
||||
clock := at("2026-10-05T22:00:00Z")
|
||||
if err := catchUpOnMerges(context.Background(), clock, stream, cat.read, cat.act(func() time.Time { return clock }),
|
||||
func(string, ...any) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat.acted) != 0 {
|
||||
t.Fatalf("a merge of three days ago was acted on: %v", cat.acted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,796 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// the catalogue: what exists, what is assigned, and how it is configured.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// provided is what comes with the control plane rather than from a repository.
|
||||
//
|
||||
// WireGuard, the names, and the domain module over both. The first two are here because the code
|
||||
// that works out their files is here:
|
||||
// a peer list is derived from every machine at once, so it cannot be written in a manifest, and
|
||||
// whatever computes it has to live wherever the whole picture is.
|
||||
//
|
||||
// **It is a module in every other respect** — assigned, unassigned, resolved, settled, and absent
|
||||
// from a machine nobody gave it to.
|
||||
func providedModules() []catalogue.Manifest {
|
||||
var out []catalogue.Manifest
|
||||
for _, raw := range []map[string]any{
|
||||
// **Two used to be here and are gone**: one wrote the mesh's names into a hosts file, the
|
||||
// other wrote the same machines as wildcards for a resolver to read. Neither ran software
|
||||
// and neither could be swapped for anything, which is the test of whether a thing is a
|
||||
// module at all (novox/hq ADR 0040). They existed because computed output needed somewhere
|
||||
// to live, and now a module says where it wants it — `facts` in its own manifest.
|
||||
//
|
||||
// **And the bundle that required it is gone** (novox/hq ADR 0226): `networking` named this
|
||||
// module's requirement and nothing else, so every machine carried two modules for one
|
||||
// network. A machine is assigned the private network itself; what a release stops shipping
|
||||
// is retired at the next start (stores.go, Inventory.RetireUnshipped).
|
||||
overlay.Manifest(),
|
||||
} {
|
||||
var m catalogue.Manifest
|
||||
b, _ := json.Marshal(raw)
|
||||
_ = json.Unmarshal(b, &m)
|
||||
out = append(out, m)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var provided = providedModules()
|
||||
|
||||
func moduleCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("module add <file>, module check <file>..., module list, or module forget <name>")
|
||||
}
|
||||
// `check` needs no mesh, and must not: it is what somebody runs in their own repository before
|
||||
// there is a mesh in reach (novox/hq issue 148). A directory expands to every manifest under it.
|
||||
if args[0] == "check" {
|
||||
var paths []string
|
||||
for _, a := range args[1:] {
|
||||
if info, err := os.Stat(a); err == nil && info.IsDir() {
|
||||
under, err := manifestsUnder(a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
paths = append(paths, under...)
|
||||
continue
|
||||
}
|
||||
paths = append(paths, a)
|
||||
}
|
||||
return moduleCheck(paths, os.Stdout)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
switch args[0] {
|
||||
case "add":
|
||||
set := flag.NewFlagSet("module add", flag.ContinueOnError)
|
||||
repo := set.String("source", "", "where this module comes from")
|
||||
ref := set.String("ref", "", "the branch followed there")
|
||||
commit := set.String("commit", "", "the commit this manifest was read at")
|
||||
// **Where inside the repository the module is** (novox/hq ADR 0069). A module is a
|
||||
// repository *and* a directory, and a record that carries only the repository names a
|
||||
// module.json at its root — so every later build of it looks in the wrong place and fails
|
||||
// with "no module.json at its root". Nine modules on this mesh were registered that way
|
||||
// and none of them could be rebuilt (2026-09-28).
|
||||
path := set.String("path", "", "the module's directory inside that repository")
|
||||
self := set.Bool("self", false, "the source is a path on the forge holding the git seat")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("module add <manifest.json> [--source <repo> [--self] [--path P] " +
|
||||
"--ref <branch> --commit <sha>]")
|
||||
}
|
||||
raw, err := os.ReadFile(positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
from, err := whereItComesFrom(*repo, *ref, *commit, *path, *self)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := namesNoInstallation(m); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, from); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s registered", m.Module)
|
||||
if *commit != "" {
|
||||
fmt.Printf(" from %s", short(*commit))
|
||||
}
|
||||
if *repo != "" && *path == "" {
|
||||
// Said, not refused: a module really at the root is the ordinary case for a repository
|
||||
// of its own. But a repository holding many modules and a record naming none of them is
|
||||
// a module nothing can rebuild, and the person adding it is the one who knows which.
|
||||
fmt.Printf("\n no directory inside %s, so it is built from that repository's root — "+
|
||||
"`--path` if the module lives in a directory there", *repo)
|
||||
}
|
||||
if len(m.Provides) > 0 {
|
||||
fmt.Printf(", providing %s", describeOffers(m.Provides))
|
||||
}
|
||||
fmt.Println()
|
||||
for _, c := range m.Claims {
|
||||
fmt.Printf(" claims %s, one per %s\n", c.Name, c.At())
|
||||
}
|
||||
return nil
|
||||
|
||||
case "list":
|
||||
// The catalogue: what exists, where it came from, whether it is current, and who runs it.
|
||||
// The provenance was recorded from the first build and nothing showed it, which made
|
||||
// "is this current?" a question you could only answer by reading the database.
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** (novox/hq ADR 0195): what each module
|
||||
// is, where it runs, whether it is current, and what it says of itself.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Built string `json:"built,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Current bool `json:"current"`
|
||||
Provided bool `json:"provided,omitempty"`
|
||||
// Tools says whether the module answers tools anywhere it runs: a list of its own,
|
||||
// a bundle the runtime serves, or a seat's verbs it claims (novox/hq ADR 0197) —
|
||||
// what the console checks the bus's answers against.
|
||||
Tools bool `json:"tools"`
|
||||
On []string `json:"on"`
|
||||
Provides []string `json:"provides,omitempty"`
|
||||
Requires []string `json:"requires,omitempty"`
|
||||
Claims []string `json:"claims,omitempty"`
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
}
|
||||
out := make([]listed, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
l := listed{Module: m.Module, Version: m.Version, Built: e.Source.BuiltFrom, Head: e.Source.Head,
|
||||
Current: e.Provided || e.Source.Repository == "" || e.Source.Current(), Provided: e.Provided,
|
||||
On: append([]string{}, e.On...), Provides: m.Offers(), Requires: m.Requires,
|
||||
Capabilities: m.Capabilities, Tools: declaresTools(m)}
|
||||
for _, c := range m.Claims {
|
||||
l.Claims = append(l.Claims, c.At()+"/"+c.Name)
|
||||
}
|
||||
out = append(out, l)
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
fmt.Println("this mesh knows about no modules yet")
|
||||
return nil
|
||||
}
|
||||
var stale int
|
||||
for _, e := range entries {
|
||||
m := e.Manifest
|
||||
fmt.Printf("%-18s %-8s", m.Module, m.Version)
|
||||
|
||||
switch {
|
||||
case e.Provided:
|
||||
fmt.Printf(" %-22s", "with the control plane")
|
||||
case e.Source.Repository == "":
|
||||
// Handed over by hand. Legitimate — it is how a module is fixed in a hurry — and
|
||||
// worth saying, because nothing can rebuild it.
|
||||
fmt.Printf(" %-22s", "handed over")
|
||||
case !e.Source.Current():
|
||||
stale++
|
||||
fmt.Printf(" %-22s", "behind "+short(e.Source.BuiltFrom)+" < "+short(e.Source.Head))
|
||||
default:
|
||||
fmt.Printf(" %-22s", "built "+short(e.Source.BuiltFrom))
|
||||
}
|
||||
|
||||
if len(e.On) > 0 {
|
||||
fmt.Printf(" on %s", strings.Join(e.On, ", "))
|
||||
} else {
|
||||
fmt.Printf(" on nothing")
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
var says []string
|
||||
if len(m.Provides) > 0 {
|
||||
says = append(says, "provides "+describeOffers(m.Provides))
|
||||
}
|
||||
if len(m.Requires) > 0 {
|
||||
says = append(says, "requires "+strings.Join(m.Requires, ", "))
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
says = append(says, "claims "+c.At()+"/"+c.Name)
|
||||
}
|
||||
if len(m.Capabilities) > 0 {
|
||||
says = append(says, "needs "+strings.Join(m.Capabilities, ", "))
|
||||
}
|
||||
if len(says) > 0 {
|
||||
fmt.Printf(" %s\n", strings.Join(says, " · "))
|
||||
}
|
||||
}
|
||||
if stale > 0 {
|
||||
fmt.Printf("\n%d module(s) behind their source — `build --behind` to catch up\n", stale)
|
||||
}
|
||||
return nil
|
||||
|
||||
case "moved":
|
||||
if len(args) != 3 {
|
||||
return errors.New("module moved <name> <commit> — the source has a newer commit")
|
||||
}
|
||||
if err := inv.SourceMoved(ctx, args[1], args[2]); err != nil {
|
||||
return err
|
||||
}
|
||||
from, err := inv.SourceOf(ctx, args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if from.Current() {
|
||||
fmt.Printf("%s is current at %s\n", args[1], short(from.Head))
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s is behind: the mesh holds %s and the source has %s\n",
|
||||
args[1], short(from.BuiltFrom), short(from.Head))
|
||||
fmt.Printf(" run `build %s` to catch up\n", from.Repository)
|
||||
return nil
|
||||
|
||||
case "forget":
|
||||
// **What goes with it is said before it goes** (novox/hq 04-ISSUES/017). The settings, the
|
||||
// module's own secrets and the ports the mesh chose all cascade off the module row, so
|
||||
// `forget` used to destroy them and report "forgotten" — an action succeeding into a state
|
||||
// its own verify would reject, and a sealed secret is not recoverable afterwards.
|
||||
set := flag.NewFlagSet("module forget", flag.ContinueOnError)
|
||||
andHeld := set.Bool("and-what-it-holds", false,
|
||||
"discard its settings, its own secrets and its ports along with it")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("module forget <name> [--and-what-it-holds]")
|
||||
}
|
||||
if !*andHeld {
|
||||
if err := inv.ForgetModule(ctx, positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s forgotten\n", positionals[0])
|
||||
return nil
|
||||
}
|
||||
held, err := inv.DiscardModule(ctx, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s forgotten\n", positionals[0])
|
||||
for _, line := range held.Lines() {
|
||||
// Said after the fact as well as before it: this is the only record that these
|
||||
// existed, and the next person to ask why the module came back empty reads it here.
|
||||
fmt.Printf(" discarded%s\n", strings.TrimPrefix(line, " "))
|
||||
}
|
||||
return nil
|
||||
|
||||
case "issue":
|
||||
// A module's broker account, scoped by its emits and consumes (novox/hq ADR 0043) and
|
||||
// sealed to the machine that will run it — the generic case the builder was the first of.
|
||||
set := flag.NewFlagSet("module issue", flag.ContinueOnError)
|
||||
forNode := set.String("node", "",
|
||||
"the machine that will run it, so the credential is delivered instead of printed")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("module issue <module> --node <machine>")
|
||||
}
|
||||
module := positionals[0]
|
||||
if *forNode == "" {
|
||||
return errors.New("module issue needs --node: a module's account is sealed to the " +
|
||||
"machine that runs it, and the mesh cannot read it back to print")
|
||||
}
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, ok := shelf[module]
|
||||
if !ok {
|
||||
return fmt.Errorf("this mesh knows no module %q; `module add` it first", module)
|
||||
}
|
||||
if err := mayIssue(m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Which bus this mesh is on. A module gets a credential for exactly one, and the two are
|
||||
// made in entirely different ways: on the bus the mesh runs on today an account is a
|
||||
// management call, and on the bus being built it is a row the next composition writes into
|
||||
// the server's user list (novox/hq design 25 §4).
|
||||
busAddress, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueOnTheNewBus(ctx, inv, m, *forNode, busAddress)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("module has no %q; it has add, check, list, moved, forget and issue", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
func assignCommand(ctx context.Context, verb string, args []string) error {
|
||||
// Several modules in one act (novox/hq ADR 0207): holders that depend on each other — the
|
||||
// service manager and the package manager — can only go on, or come off, together.
|
||||
if len(args) < 2 {
|
||||
return fmt.Errorf("%s <node> <module> [<module>…]", verb)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
// The act itself is in acts.go, so the command API refuses exactly what this refuses
|
||||
// (novox/hq ADR 0035). What differs between the surfaces is how the answer is printed.
|
||||
act := assign
|
||||
if verb == "unassign" {
|
||||
act = unassign
|
||||
}
|
||||
said, err := act(ctx, open, args[0], args[1:]...)
|
||||
if said != "" {
|
||||
fmt.Println(said)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func settingsCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("settings set <module> <file> [--node <node>], or settings clear <module> [--node <node>]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
set := flag.NewFlagSet("settings", flag.ContinueOnError)
|
||||
node := set.String("node", "", "one machine, rather than the whole mesh")
|
||||
positionals, err := parseAround(set, args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
where := "the whole mesh"
|
||||
if *node != "" {
|
||||
where = *node
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "set":
|
||||
if len(positionals) != 2 {
|
||||
return errors.New("settings set <module> <settings.json | {…}> [--node <node>]")
|
||||
}
|
||||
// A file, or the values themselves when they begin with `{` — which is how the mesh's own
|
||||
// `settings` tool passes them, having no file to hand over (novox/hq issue 198).
|
||||
var raw []byte
|
||||
if strings.HasPrefix(strings.TrimSpace(positionals[1]), "{") {
|
||||
raw = []byte(positionals[1])
|
||||
} else if raw, err = os.ReadFile(positionals[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
var values map[string]any
|
||||
if err := json.Unmarshal(raw, &values); err != nil {
|
||||
return fmt.Errorf("%s is not a settings file: %w", positionals[1], err)
|
||||
}
|
||||
if err := inv.SetSettings(ctx, *node, positionals[0], values); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var keys []string
|
||||
for k := range values {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Printf("%s on %s: %s\n", positionals[0], where, strings.Join(keys, ", "))
|
||||
fmt.Println(" run `push` to send it")
|
||||
return nil
|
||||
|
||||
case "clear":
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("settings clear <module> [--node <node>]")
|
||||
}
|
||||
if err := inv.ClearSettings(ctx, *node, positionals[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s is back to what the module says\n", positionals[0], where)
|
||||
return nil
|
||||
|
||||
default:
|
||||
return fmt.Errorf("settings has no %q; it has set and clear", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// describeOffers says what a module provides, and marks the ones answered from anywhere in the
|
||||
// mesh — because "provides a database" and "provides a shell" are read the same way and mean
|
||||
// entirely different things about where the answer has to be.
|
||||
func describeOffers(offers []catalogue.Offer) string {
|
||||
var out []string
|
||||
for _, o := range offers {
|
||||
if o.At() == catalogue.ScopeMesh {
|
||||
out = append(out, o.Name+" (from anywhere in the mesh)")
|
||||
continue
|
||||
}
|
||||
out = append(out, o.Name)
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// pinCommand says which node a machine gets a provision from.
|
||||
//
|
||||
// Needed only when more than one could answer, and recordable before that -- a mesh with one
|
||||
// database should not change where an existing machine gets its data the day a second one
|
||||
// arrives.
|
||||
func pinCommand(ctx context.Context, args []string, setting bool) error {
|
||||
if setting && len(args) != 4 {
|
||||
return errors.New("pin <node> <provision> <from-node> <module>")
|
||||
}
|
||||
if !setting && len(args) != 2 {
|
||||
return errors.New("unpin <node> <provision>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if !setting {
|
||||
if err := inv.UnpinProvision(ctx, args[0], args[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is no longer told where to get %s from\n", args[0], args[1])
|
||||
return nil
|
||||
}
|
||||
// The provider's node may be this same machine: two modules beside the consumer can both
|
||||
// answer a provision, and then the module is the whole question (novox/hq #258).
|
||||
if err := inv.PinProvision(ctx, args[0], args[1], args[2], args[3]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s gets %s from %s/%s\n", args[0], args[1], args[2], args[3])
|
||||
fmt.Printf(" run `push %s` to send it\n", args[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
// theBrokerSeat is the mesh-scoped seat the broker module claims (novox/hq ADR 0079: a
|
||||
// foundation seat is named after the server it guards).
|
||||
const theBrokerSeat = "mesh-broker"
|
||||
|
||||
// brokerReachableAt is the broker's address as the given node can reach it.
|
||||
//
|
||||
// The genesis address (MESH_BROKER_ADDRESS) is the broker's public endpoint — right for a machine
|
||||
// that can dial it, wrong across the mesh, where the path is the overlay. A node that is ON the
|
||||
// overlay is given the overlay name of the node HOLDING the broker — the one assigned a module
|
||||
// claiming the mesh-broker seat — because that is where the broker is, whatever else the topology
|
||||
// says. Only when nothing holds the seat yet (genesis raised the broker as plumbing and no module
|
||||
// has adopted it) does the hub stand in, which is where the foundation is by convention.
|
||||
//
|
||||
// "On the overlay" is what `whereEveryoneIs` answers — a machine that RESOLVED the private network
|
||||
// — not "has an address", which is true of every placed machine and says nothing about
|
||||
// whether anything can reach it (novox/hq issue 059). A node not on the overlay — at genesis,
|
||||
// before any `overlay place`, which is when the builder's account is issued — keeps the genesis
|
||||
// address, so nothing about bring-up changes. This is issue 055, corrected by 059.
|
||||
func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known broker.Broker, node string) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
// No catalogue yet is genesis, and at genesis the genesis address is the right one.
|
||||
return known.Address, nil
|
||||
}
|
||||
if onNetwork[node] == "" {
|
||||
return known.Address, nil
|
||||
}
|
||||
|
||||
// Where the broker is: the node whose assigned set includes a module claiming the seat.
|
||||
holders := map[string]bool{}
|
||||
for name, m := range shelf {
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == theBrokerSeat && c.At() == catalogue.ScopeMesh {
|
||||
holders[name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
brokerAt := ""
|
||||
if len(holders) > 0 {
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, o := range overlays {
|
||||
assigned, err := inv.Assigned(ctx, o.Name)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, a := range assigned {
|
||||
if holders[a] && onNetwork[o.Name] != "" {
|
||||
brokerAt = onNetwork[o.Name]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if brokerAt == "" {
|
||||
// Nothing holds the seat (or its node is not on the overlay): the hub, where the
|
||||
// foundation is by convention — and only a hub that is itself on the overlay, or the
|
||||
// name we hand out routes nowhere.
|
||||
overlays, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, o := range overlays {
|
||||
if o.Hub && onNetwork[o.Name] != "" {
|
||||
brokerAt = onNetwork[o.Name]
|
||||
}
|
||||
}
|
||||
}
|
||||
if brokerAt == "" {
|
||||
return known.Address, nil
|
||||
}
|
||||
// A portless genesis address is a working configuration (amqps defaults to 5671), and
|
||||
// silently keeping the public address would disable this whole path — so the port defaults
|
||||
// rather than the fix dissolving (novox/hq issue 059).
|
||||
_, port, err := net.SplitHostPort(known.Address)
|
||||
if err != nil {
|
||||
port = "5671"
|
||||
}
|
||||
return net.JoinHostPort(brokerAt, port), nil
|
||||
}
|
||||
|
||||
// mayIssue says whether a module can be given a broker account. The account is delivered as the
|
||||
// module's own secret named broker; a module that declares none has nothing to read it with, and
|
||||
// an account nothing reads is an orphan on the bus (novox/hq 04-ISSUES/078). Said before the
|
||||
// account is made, not after.
|
||||
func mayIssue(m catalogue.Manifest) error {
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||
//
|
||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||
// is minted into the mesh's records and becomes usable at the next composition, so there is no
|
||||
// server to be reachable for this to work. The password travels beside the address rather than inside
|
||||
// it, because the runtime's contract already separates them and a credential embedded in a URL is one
|
||||
// that leaks into every log line that prints a connection. And the module's durable consumer is
|
||||
// derived from what it declared rather than declared by name, so a module cannot ask for delivery of
|
||||
// something it did not say it consumes.
|
||||
func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||
node, busAddress string) error {
|
||||
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: node, Module: m.Module}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: busKindOf(m.Module), Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Where the module is told to find the bus, and what certificate it must present. The same pair
|
||||
// a node is told, for the same reason: a mesh's bus presents its own certificate, in no public
|
||||
// trust store, so an address alone fails at TLS.
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot deliver a credential without knowing where the bus is: %w", err)
|
||||
}
|
||||
reachable, err := brokerReachableAt(ctx, inv, known, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return issueWith(ctx, inv, m, node, busAddress, known, reachable, user, password)
|
||||
}
|
||||
|
||||
// busKindOf is what a module's bus user is recorded as: the node's tool runtime where the module is
|
||||
// the runtime (novox/hq ADR 0175), a module otherwise. The username is the same either way — the
|
||||
// runtime is issued through this same path — and the kind is what a reader of the records sees.
|
||||
func busKindOf(module string) string {
|
||||
if module == catalogue.RuntimeModule {
|
||||
return inventory.BusNodeTools
|
||||
}
|
||||
return inventory.BusModule
|
||||
}
|
||||
|
||||
// issueWith is the delivery half: the minted password sealed to the machine as the module's broker
|
||||
// secret, and the module's consumer created where the bus can be reached. Split from the minting
|
||||
// so the move can issue every module against a bus whose address it worked out itself
|
||||
// (`rollout mint`, design 28 task 5.2) rather than the one in this process's environment.
|
||||
func issueWith(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest,
|
||||
node, busAddress string, known broker.Broker, reachable, user, password string) error {
|
||||
// The seats this module claims, with the verbs each promises (novox/hq ADR 0159): the runtime
|
||||
// serves a claimed seat's verbs with its tools of the same name, and the bus admits only the
|
||||
// holder's subscription — so the runtime tries each claim and the grant decides. Written here
|
||||
// because this file is the one thing the mesh writes that the runtime reads before it speaks.
|
||||
claims, err := claimsFor(ctx, inv, m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Claims []seatClaimed `json:"claims,omitempty"`
|
||||
}{
|
||||
URL: "nats://" + reachable, Fingerprint: known.Fingerprint,
|
||||
Node: node, Module: m.Module, User: user, Password: password, Claims: claims,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, "broker", string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// And how it hears what it consumes. Derived from its declaration, and only when it declared
|
||||
// something: a module that consumes nothing needs no consumer, and creating one would be a
|
||||
// durable subscription nobody reads.
|
||||
if consumer, needed := broker.ConsumerFor(broker.Principal{
|
||||
Kind: broker.KindModule, Node: node, Module: m.Module,
|
||||
Emits: m.EmitsAll(), Consumes: m.Consumes, Serves: m.Tools,
|
||||
}); needed {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s consumes; its consumer is created when the bus is reachable (`push`, then "+
|
||||
"`rollout mint` again is harmless)\n", m.Module)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||
"how %s hears what it consumes: %w", m.Module, err)
|
||||
}
|
||||
defer js.Close()
|
||||
if err := js.EnsureConsumer(consumer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("bus user %s minted for %s, scoped to what it emits and consumes\n", user, m.Module)
|
||||
fmt.Printf(" sealed to %s. It arrives with the next push — `push %s` to send it\n", node, node)
|
||||
fmt.Printf(" and it works once the bus has been told: the user list is composed into the " +
|
||||
"machine holding mesh-broker\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// whereItComesFrom is the provenance a module handed over by hand records, and what a record must
|
||||
// say to be worth anything later.
|
||||
//
|
||||
// **A module is a repository and a directory inside it** (novox/hq ADR 0069). A record carrying only
|
||||
// the repository names a module.json at its root, so every later build of it looks in the wrong
|
||||
// place — nine modules on this mesh were registered that way and none of them could be rebuilt
|
||||
// (2026-09-28). The directory cannot be checked from here, because the control plane does not clone;
|
||||
// what can be checked is that the record is whole.
|
||||
func whereItComesFrom(repository, ref, commit, path string, self bool) (inventory.Source, error) {
|
||||
// Provenance together or not at all. A source with no commit cannot be compared against
|
||||
// anything, so it would record where the module came from and still never be able to say the
|
||||
// mesh is behind it — which is the one thing recording it is for.
|
||||
if (repository == "") != (commit == "") {
|
||||
return inventory.Source{}, errors.New("--source and --commit go together: a source with " +
|
||||
"no commit cannot be compared against anything, and a commit with no source has " +
|
||||
"nothing to be compared with")
|
||||
}
|
||||
// A directory or a forge with no repository is half a location, and the half it keeps is the
|
||||
// half nothing can be found with.
|
||||
if repository == "" && (path != "" || self) {
|
||||
return inventory.Source{}, errors.New("--path and --self say where inside a source and " +
|
||||
"which forge holds it, so they need --source: without one there is nothing for them " +
|
||||
"to be part of")
|
||||
}
|
||||
from := inventory.Source{Repository: repository, Ref: ref, BuiltFrom: commit, Path: path}
|
||||
if self {
|
||||
if err := onASeat(repository); err != nil {
|
||||
return inventory.Source{}, err
|
||||
}
|
||||
from.Seat = gitSeat
|
||||
}
|
||||
return from, nil
|
||||
}
|
||||
|
||||
// namesNoInstallation is the mesh refusing a definition that names an installation, at the moment
|
||||
// it would enter the catalogue (novox/hq ADR 0112, ADR 0155). `module check` says the same thing
|
||||
// earlier, where the author is; this is the last moment the mesh can still say no, and a
|
||||
// definition that got past the check — written elsewhere, or checked by nobody — is refused here
|
||||
// in the same words. A name meant on purpose is declared with its reason and passes.
|
||||
func namesNoInstallation(m catalogue.Manifest) error {
|
||||
named := catalogue.InstallationProblems(m)
|
||||
if len(named) == 0 {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s names an installation, and a definition names none — declare a name meant "+
|
||||
"on purpose under %s with its reason, or take it out:\n - %s",
|
||||
m.Module, catalogue.NamesOnPurpose, strings.Join(named, "\n - "))
|
||||
}
|
||||
|
||||
// seatClaimed is one seat a module claims, as its runtime needs it: the name, the scope (a
|
||||
// node-scoped seat's verb carries the machine, design 33 §4) and the verbs the seat promises.
|
||||
type seatClaimed struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Serves []string `json:"serves,omitempty"`
|
||||
}
|
||||
|
||||
// claimsFor joins a module's claims with the seats' protocols from the mesh's records.
|
||||
func claimsFor(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest) ([]seatClaimed, error) {
|
||||
if len(m.Claims) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
seats, err := inv.Seats(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byName := map[string]catalogue.Seat{}
|
||||
for _, s := range seats {
|
||||
byName[s.Name] = s
|
||||
}
|
||||
var out []seatClaimed
|
||||
for _, c := range m.Claims {
|
||||
claimed := seatClaimed{Seat: c.Name, Scope: c.At()}
|
||||
if s, known := byName[c.Name]; known {
|
||||
claimed.Scope = s.Scope
|
||||
// The verbs the runtime serves for the seat: the claim's own when it names them
|
||||
// (ADR 0160), else every verb the seat promises, which its tools then answer.
|
||||
claimed.Serves = c.ServesFor(catalogue.Manifest{Tools: catalogue.VerbNames(s.Serves)})
|
||||
}
|
||||
out = append(out, claimed)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// declaresTools is whether a module answers tools wherever it runs (novox/hq ADR 0197): it names
|
||||
// tools of its own, its build delivers a bundle the node's runtime serves, or it claims a seat
|
||||
// whose verbs it serves. A module with none is never expected to announce anything.
|
||||
func declaresTools(m catalogue.Manifest) bool {
|
||||
if len(m.Tools) > 0 {
|
||||
return true
|
||||
}
|
||||
for _, b := range m.Bundles {
|
||||
if len(b.Loads) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if len(c.Serves) > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// `module issue` makes a broker account and delivers it as the module's own secret named broker.
|
||||
// A module that declares none is refused before the account exists, so the bus never carries an
|
||||
// account nothing reads (novox/hq 04-ISSUES/078).
|
||||
func TestAModuleWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||
err := mayIssue(catalogue.Manifest{Module: "step-ca", OwnSecrets: catalogue.OwnSecrets{"password": {Path: "/run/password"}}})
|
||||
if err == nil {
|
||||
t.Fatal("a module with no broker own secret was issued an account")
|
||||
}
|
||||
for _, want := range []string{"step-ca", "broker", "own-secrets"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if err := mayIssue(catalogue.Manifest{Module: "redis", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}}); err != nil {
|
||||
t.Errorf("a module declaring its broker secret was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,732 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// the private network: who is on it, where, and what they are called.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// DefaultOverlayCIDR is the range the mesh allocates from when nothing says another.
|
||||
const DefaultOverlayCIDR = "10.42.0.0/16"
|
||||
|
||||
// overlayRange is the range the mesh allocates node addresses from.
|
||||
//
|
||||
// **The adopted tunnel's range first** (novox/hq ADR 0105): a hub that took over the tunnel it
|
||||
// found is at that tunnel's address, its peers are at theirs, and every node's address is
|
||||
// composed from the same range — the hub's, and every binding, hosts entry and endpoint derived
|
||||
// from it. Those are readers of this; none of them stores the range. Without an adopted tunnel,
|
||||
// the range genesis was told, or the default.
|
||||
func overlayRange(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
tunnel, _, adopted, err := inv.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if adopted {
|
||||
return tunnel.Range, nil
|
||||
}
|
||||
if v := strings.TrimSpace(os.Getenv(OverlayCIDRVar)); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
return DefaultOverlayCIDR, nil
|
||||
}
|
||||
|
||||
func overlayCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("overlay place <node> [flags], overlay name <address> <name>, or overlay show")
|
||||
}
|
||||
// Answered before anything is opened. A message about which command to use should not need a
|
||||
// database to say so, and needing one turns a redirect into a connection error.
|
||||
if args[0] == "push" {
|
||||
return errors.New("`overlay push` is now `push`, which sends a node its network AND " +
|
||||
"what its assignments resolve to — the two are computed from one picture of the " +
|
||||
"mesh, and sending them separately would let them disagree")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
switch args[0] {
|
||||
case "place":
|
||||
return overlayPlace(ctx, inv, args[1:])
|
||||
case "show":
|
||||
return overlayShow(ctx, open)
|
||||
case "name":
|
||||
return overlayName(ctx, inv, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("overlay has no %q; it has place, name and show", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// overlayName is the operator saying which machine a carried address is (novox/hq issue 112),
|
||||
// so the mesh answers for its name until the machine enrols and verifies it.
|
||||
func overlayName(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("overlay name <carried-address> <node-name>")
|
||||
}
|
||||
address, name := args[0], args[1]
|
||||
if err := inv.NamePeer(ctx, address, name); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("the peer at %s is %s until it enrols — the mesh answers for %s.<suffix> from the "+
|
||||
"operator's word, and enrolment under this key must use this name\n", address, name, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func overlayPlace(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(
|
||||
"overlay place <node> [--endpoint host:port] [--site name] [--hub], or --nothing")
|
||||
}
|
||||
node := args[0]
|
||||
|
||||
set := flag.NewFlagSet("overlay place", flag.ContinueOnError)
|
||||
endpoint := set.String("endpoint", "", "where this node can be dialled, or empty for nowhere")
|
||||
site := set.String("site", "", "where this machine physically is, or empty if it roams")
|
||||
hub := set.Bool("hub", false, "this node is the hub every other routes through")
|
||||
nothing := set.Bool("nothing", false,
|
||||
"place it with nothing set: not dialable, no site, not the hub")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// **All three are declared together, so saying nothing took all three away.** The sibling of
|
||||
// `node public-domain`: `overlay place anchor` reads like it places the node it names, and it
|
||||
// silently unset the endpoint every other machine dials, the site it is in, and the hub if it
|
||||
// was the hub — every path through it going with them, at the moment somebody was trying to
|
||||
// look at it.
|
||||
//
|
||||
// A placement with nothing set is a real thing to want — a machine that roams and opens every
|
||||
// path itself is exactly that — so it keeps a way to say so, by name.
|
||||
if set.NFlag() == 0 {
|
||||
return fmt.Errorf("overlay place %s was given nothing to place it with, and all three are "+
|
||||
"declared together — it would take away the endpoint other machines dial %s at, its "+
|
||||
"site, and the hub if it is the hub. Say --endpoint/--site/--hub, or --nothing if that "+
|
||||
"is what you meant", node, node)
|
||||
}
|
||||
if *nothing && (*endpoint != "" || *site != "" || *hub) {
|
||||
return fmt.Errorf("give %s a placement or --nothing, not both: they say opposite things "+
|
||||
"and the mesh will not choose between them", node)
|
||||
}
|
||||
|
||||
// One hub per mesh, refused rather than last-write-wins: with two flagged, which one the
|
||||
// graph and the broker address pick is order-dependent — the silent-election fault ADR 0007
|
||||
// exists to avoid, one flag over (novox/hq issue 059). Moving the hub is explicit: re-place
|
||||
// the old one without --hub first.
|
||||
if *hub {
|
||||
placed, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, p := range placed {
|
||||
if p.Hub && p.Name != node {
|
||||
return fmt.Errorf("%s is already the hub; a mesh has one. Re-place %s without "+
|
||||
"--hub first if the hub is moving", p.Name, p.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A hub that took over a tunnel listens on that tunnel's port — it is what the peers dial, and
|
||||
// the reason the port is worth having (novox/hq ADR 0105). An endpoint on another port would
|
||||
// have the mesh's interface up where no peer is listening for it.
|
||||
found, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var tunnel inventory.Tunnel
|
||||
adoptsTunnel := false
|
||||
if *hub && found.Adopted {
|
||||
if t, err := inv.TunnelOf(ctx, node); err == nil {
|
||||
tunnel = t
|
||||
placed, _ := inv.Overlays(ctx)
|
||||
for _, o := range placed {
|
||||
if o.Name == node && o.Key == t.PublicKey {
|
||||
adoptsTunnel = true
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, inventory.ErrNoTunnel) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if adoptsTunnel {
|
||||
if port := portOfEndpoint(*endpoint); port != strconv.Itoa(tunnel.Port) {
|
||||
return fmt.Errorf("%s takes over the tunnel it found on %s, which listens on port %d, and "+
|
||||
"its endpoint %q names another port: the peers dial the tunnel's port, so the hub's "+
|
||||
"endpoint must be on it", node, tunnel.Interface, tunnel.Port, *endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Declared, all three. The address is evidence of reachability and is not the fact, and hub
|
||||
// election by address prefix fails silently (novox/hq ADR 0007).
|
||||
if err := inv.SetPlace(ctx, node, *endpoint, *site, *hub, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
address, err := inv.AssignAddress(ctx, found.ID, cidr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("%s is at %s on the overlay\n", node, address)
|
||||
fmt.Println(" credentials issued for it before this placement keep their old broker address —" +
|
||||
" `module issue` them again and push (novox/hq issue 059)")
|
||||
switch {
|
||||
case adoptsTunnel:
|
||||
fmt.Printf(" the hub — it takes over the tunnel it found on %s: range %s, port %d, "+
|
||||
"%d peer(s) carried until they enrol\n", tunnel.Interface, tunnel.Range, tunnel.Port,
|
||||
len(tunnel.Peers))
|
||||
case *hub:
|
||||
fmt.Println(" the hub — every node not sharing a site routes through it")
|
||||
case *endpoint == "":
|
||||
fmt.Println(" not dialable — it opens every path itself")
|
||||
}
|
||||
if *site != "" {
|
||||
fmt.Printf(" at %s, so it peers directly with anything else there\n", *site)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// network builds the private network over the machines that resolved the module for it.
|
||||
//
|
||||
// Not over every node the mesh knows. **A machine is on the private network because it was given
|
||||
// the module**, and one that was not is absent from every peer list and from the names — which is
|
||||
// the only thing "not on the network" can mean. Until this, having an address was enough, and
|
||||
// there was no way to keep a machine off.
|
||||
//
|
||||
// Every node at once, which is the whole reason this is the control plane's work: a peer list is
|
||||
// derived from all the others, so no node could compute its own.
|
||||
func network(ctx context.Context, inv *inventory.Inventory, on map[string]bool,
|
||||
refused map[string]string) (*overlay.Generator, error) {
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The tunnels adopted nodes take over, and the peers the hub's carries (novox/hq ADR 0105).
|
||||
tunnels, err := inv.Tunnels(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nodes := make([]overlay.Node, 0, len(places))
|
||||
for _, p := range places {
|
||||
if !on[p.Name] {
|
||||
continue
|
||||
}
|
||||
n := overlay.Node{
|
||||
Name: p.Name, Key: p.Key, Endpoint: p.Endpoint,
|
||||
Site: p.Site, Hub: p.Hub, Address: p.Address,
|
||||
}
|
||||
if t, takes := tunnels[p.Name]; takes && t.NodeAdopted {
|
||||
// Only an adopted node is told to take the found unit over: on a converged one there
|
||||
// is nothing found to keep, and the host refuses the field. The range and the carried
|
||||
// peers do not depend on the mode; the takeover does.
|
||||
//
|
||||
// **Refused, not composed, when the hub's record disagrees with the tunnel.** A
|
||||
// declaration that stopped the found unit and raised the mesh's interface on another
|
||||
// port or address would leave every peer dark while reporting the tunnel taken — so a
|
||||
// hub placed before it took the tunnel over (or at the wrong port) is named here, and
|
||||
// nothing is sent until it is re-placed.
|
||||
if wrong := disagrees(p, t.Tunnel); wrong != "" {
|
||||
return nil, fmt.Errorf("%s takes over the tunnel on %s and its placement disagrees with it: %s. "+
|
||||
"Re-place it — `overlay place %s --hub --endpoint <host>:%d …` — and push again; "+
|
||||
"nothing was composed", p.Name, t.Interface, wrong, p.Name, t.Port)
|
||||
}
|
||||
n.TakesOver = &overlay.TakeOver{Interface: t.Interface, Unit: t.Unit, Config: t.Config, Port: t.Port, MTU: t.MTU}
|
||||
}
|
||||
if p.Hub {
|
||||
for _, c := range carried {
|
||||
n.Carried = append(n.Carried, overlay.Carried{Key: c.PublicKey, Address: c.Address})
|
||||
}
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Nobody was given it. An empty network is a legitimate mesh, not a broken one, so this
|
||||
// answers rather than refusing -- Compute would refuse for want of a hub, and reporting
|
||||
// "no hub" to somebody who never asked for a network would be a lie about the cause.
|
||||
return overlay.Empty(), nil
|
||||
}
|
||||
cidr, err := overlayRange(ctx, inv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No registry trust is composed here any more: the container runtime's module states it, told
|
||||
// where the store is reached by ${seat:mesh-artifact-store:reach} (novox/hq ADR 0222, issue 190).
|
||||
g, err := overlay.From(nodes, cidr, "")
|
||||
if err != nil && len(refused) > 0 {
|
||||
// The network is missing something, and some machines could not be resolved at all. Those
|
||||
// are almost always the same fact: a node that does not resolve contributes nothing, so
|
||||
// reporting "no hub" would name a consequence and hide the cause.
|
||||
var who []string
|
||||
for name, why := range refused {
|
||||
who = append(who, fmt.Sprintf(" %s: %s", name, why))
|
||||
}
|
||||
sort.Strings(who)
|
||||
return nil, fmt.Errorf("%w\n\nand %d node(s) could not be resolved at all, which is "+
|
||||
"probably why:\n%s", err, len(refused), strings.Join(who, "\n"))
|
||||
}
|
||||
return g, err
|
||||
}
|
||||
|
||||
// graph is the whole mesh's network, for showing it.
|
||||
func graph(ctx context.Context, open *stores) ([]overlay.Node, overlay.Graph, error) {
|
||||
inv := open.inventory
|
||||
on, refused, err := whoResolves(ctx, open, overlay.Requirement)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
g, err := network(ctx, inv, on, refused)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return g.Nodes(), g.Graph(), nil
|
||||
}
|
||||
|
||||
// whoResolves is the machines whose resolution answers a requirement, and why the others did not.
|
||||
//
|
||||
// By what a module **provides**, not by its name. WireGuard is one way to have a private network
|
||||
// and there could be others, so a machine is on the network because something it runs provides
|
||||
// one — asking for a particular module by name would be the mistake this whole mechanism exists
|
||||
// to avoid.
|
||||
//
|
||||
// Resolved rather than read from the assignment table, because a module can arrive by being
|
||||
// required by something else, and a machine that needs the private network to do its job is on it
|
||||
// for the same reason as one that was handed it directly.
|
||||
func whoResolves(ctx context.Context, open *stores, requirement string) (
|
||||
map[string]bool, map[string]string, error) {
|
||||
inv := open.inventory
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
on := map[string]bool{}
|
||||
// Why a node could not be resolved, kept rather than raised: one broken node must not stop
|
||||
// the rest being described, and whoever is rendering that node will raise it themselves.
|
||||
refused := map[string]string{}
|
||||
for _, n := range nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
switch {
|
||||
case unresolvable(err):
|
||||
refused[n.Name] = err.Error()
|
||||
continue
|
||||
case err != nil:
|
||||
// Not a node that does not resolve — a question that went unanswered. Recording it as a
|
||||
// refusal would take the machine off the private network, and the generator that reads
|
||||
// this would then write a roster and a filter without it (novox/hq 04-ISSUES/152).
|
||||
return nil, nil, fmt.Errorf("whether %s answers %q cannot be read: %w",
|
||||
n.Name, requirement, err)
|
||||
}
|
||||
for _, m := range plan.Modules {
|
||||
for _, offered := range m.Offers() {
|
||||
if offered == requirement {
|
||||
on[n.Name] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return on, refused, nil
|
||||
}
|
||||
|
||||
// rendering is everything a declaration needs, computed over the whole mesh.
|
||||
func generators(ctx context.Context, open *stores) (
|
||||
map[string]catalogue.Generator, error) {
|
||||
inv := open.inventory
|
||||
on, refused, err := whoResolves(ctx, open, overlay.Addressing)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
net, err := network(ctx, inv, on, refused)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **One generator now.** Two more used to sit beside it — the names and a resolver's zone
|
||||
// file — as modules that ran nothing. Both are facts a module asks for in its manifest
|
||||
// (`facts:` — catalogue.FactsInto), computed from the same machines this sees: the ones on the
|
||||
// private network, because a name for a machine not on it would resolve to an address nothing
|
||||
// can reach.
|
||||
return map[string]catalogue.Generator{
|
||||
overlay.Name: net,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func overlayShow(ctx context.Context, open *stores) error {
|
||||
nodes, computed, err := graph(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Not "this mesh has no nodes", which it said until the network became a module and was
|
||||
// then a lie about the cause: a mesh can have every node it will ever have and nobody on
|
||||
// the private network, because nobody asked for one.
|
||||
fmt.Printf("nobody is on the private network — assign %s to put a machine on it\n",
|
||||
overlay.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// The tunnel the hub took over, if any, and the peers carried from it (novox/hq ADR 0105):
|
||||
// listed apart from the nodes, because they are peers of the tunnel and not nodes of the
|
||||
// mesh until they enrol — and once one has, it is listed as the node it became.
|
||||
tunnel, hubName, adopted, err := open.inventory.AdoptedTunnel(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
carried, err := open.inventory.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, n := range nodes {
|
||||
place := n.Address
|
||||
if place == "" {
|
||||
// Said, not skipped. A node with no place is a node with no network, and it should
|
||||
// be visible here rather than quietly absent from a list of who is on it.
|
||||
place = "no address — run `overlay place`"
|
||||
}
|
||||
fmt.Printf("%-16s %-14s", n.Name, place)
|
||||
switch {
|
||||
case n.Hub && adopted:
|
||||
fmt.Printf(" hub — over the tunnel it took over on %s (range %s, port %d)",
|
||||
tunnel.Interface, tunnel.Range, tunnel.Port)
|
||||
case n.Hub && hubName == n.Name && tunnel.Interface != "":
|
||||
fmt.Printf(" hub — found a tunnel on %s and did NOT take it over: its key is not the tunnel's; "+
|
||||
"`mesh-host overlay take --tunnel %s` on the machine takes it", tunnel.Interface, tunnel.Interface)
|
||||
case n.Hub:
|
||||
fmt.Print(" hub — found no tunnel; if the machine runs the predecessor's, " +
|
||||
"`mesh-host overlay take --tunnel <iface>` there adopts it (novox/hq ADR 0105)")
|
||||
case !n.Reachable():
|
||||
fmt.Print(" not dialable")
|
||||
}
|
||||
if n.Site != "" {
|
||||
fmt.Printf(" at %s", n.Site)
|
||||
}
|
||||
if n.TakesOver != nil && !n.Hub {
|
||||
fmt.Printf(" takes over %s", n.TakesOver.Interface)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, p := range computed[n.Name] {
|
||||
fmt.Printf(" → %-14s %-18s %s\n", p.Name, p.Allowed, p.Why)
|
||||
}
|
||||
}
|
||||
if len(carried) > 0 {
|
||||
fmt.Printf("\npeers of the tunnel %s took over — not nodes of the mesh until they enrol:\n", hubName)
|
||||
for _, c := range carried {
|
||||
state := "not yet enrolled"
|
||||
if c.EnrolledAs != "" {
|
||||
state = "enrolled as " + c.EnrolledAs + ", which keeps this address"
|
||||
}
|
||||
fmt.Printf(" %-16s %-14s %s\n", overlay.CarriedName(c.PublicKey), c.Address, state)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// disagrees says how a node's placement differs from the tunnel it takes over — its address not
|
||||
// the tunnel's, its endpoint not on the tunnel's port — or nothing when both agree.
|
||||
func disagrees(p inventory.Overlay, t inventory.Tunnel) string {
|
||||
var wrong []string
|
||||
want := t.Address
|
||||
if i := strings.Index(want, "/"); i >= 0 {
|
||||
want = want[:i]
|
||||
}
|
||||
if p.Address != want {
|
||||
wrong = append(wrong, fmt.Sprintf("its address is %s and the tunnel's is %s", orNothing(p.Address), want))
|
||||
}
|
||||
if p.Reachable() && portOfEndpoint(p.Endpoint) != strconv.Itoa(t.Port) {
|
||||
wrong = append(wrong, fmt.Sprintf("its endpoint %s is not on the tunnel's port %d", p.Endpoint, t.Port))
|
||||
}
|
||||
return strings.Join(wrong, "; ")
|
||||
}
|
||||
|
||||
func orNothing(s string) string {
|
||||
if s == "" {
|
||||
return "unset"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// portOfEndpoint is the port in host:port, or empty.
|
||||
func portOfEndpoint(endpoint string) string {
|
||||
if i := strings.LastIndex(endpoint, ":"); i >= 0 {
|
||||
return endpoint[i+1:]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SilentFor is how long a node may be quiet before the mesh says so.
|
||||
//
|
||||
// A node speaks every minute, so three of them missed is a gap rather than a slow one. The number
|
||||
// is not the point — being able to say "out of touch" at all is, and nothing could before.
|
||||
const SilentFor = 3 * time.Minute
|
||||
|
||||
// whereEveryoneIs is each machine's name on the private network, for the ones on it.
|
||||
//
|
||||
// **Resolved without consulting the rest of the mesh**, and that is not an optimisation. Every
|
||||
// other path here answers a question about one node by resolving the others; this one is called
|
||||
// *from* that path, so doing the same would not terminate — which it did not, for two minutes,
|
||||
// until it was run.
|
||||
//
|
||||
// An unchecked resolution is exactly right for the question anyway. Whether a machine is on the
|
||||
// private network depends on what it was assigned and what that requires, both of which are local
|
||||
// facts. What it takes *from* other machines does not change the answer.
|
||||
//
|
||||
// The distinction that matters is kept: a machine absent from the network module's own view is
|
||||
// absent here, so "has an address" is not mistaken for "is reachable" — which it was, before the
|
||||
// network became something a machine is given.
|
||||
func whereEveryoneIs(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||
|
||||
if shelf == nil {
|
||||
// Refused rather than answered. Being on the private network is a conclusion about what a
|
||||
// node resolves to, so with no catalogue nothing resolves and the honest answer is
|
||||
// "nobody" — which is wrong, indistinguishable from a mesh with no overlay, and refused
|
||||
// every certificate the mesh was asked for while saying the machine was on no network.
|
||||
return nil, errors.New(
|
||||
"asked where everyone is without the catalogue, which cannot be answered")
|
||||
}
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, p := range places {
|
||||
out[p.Name] = overlay.InternalName(p.Name)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onTheNetwork is every placed machine that resolves the private network — has an address AND
|
||||
// runs what puts it there. "Has an address" alone was true of every placed machine and told you
|
||||
// nothing about whether anything could reach it; a name written for such a machine resolves to
|
||||
// an address that does not answer, and a connection to it hangs (novox/hq issue 079).
|
||||
func onTheNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) ([]inventory.Overlay, error) {
|
||||
places, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// **With the seat holders on record**, or a machine running the next holder of a seat beside
|
||||
// the current one resolves as two holders, is refused, and drops out of the map — taking the
|
||||
// address every other machine composes for what it offers (novox/hq ADR 0131). Found live:
|
||||
// the control node vanished from the private network the moment the new bus was assigned
|
||||
// beside the old one.
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.Overlay
|
||||
for _, p := range places {
|
||||
if p.Address == "" {
|
||||
continue
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, p.Name)
|
||||
if err != nil || len(assigned) == 0 {
|
||||
continue
|
||||
}
|
||||
caps, _ := inv.ProfileOf(ctx, p.Name)
|
||||
got, err := catalogue.Resolve(shelf, assigned,
|
||||
catalogue.Node{Name: p.Name, Site: p.Site, Capabilities: caps},
|
||||
catalogue.World{Unchecked: true, Holdings: holdings})
|
||||
if err != nil {
|
||||
// Said, not skipped in silence: a machine dropped here loses its address, and every
|
||||
// plan that names it fails in another module's words (novox/hq issue 188).
|
||||
fmt.Fprintf(os.Stderr, "%s is not counted as on the network: it does not resolve: %v\n", p.Name, err)
|
||||
continue
|
||||
}
|
||||
for _, m := range got.Modules {
|
||||
for _, offered := range m.Offers() {
|
||||
if offered == overlay.Requirement {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onThePrivateNetwork is every node's address on the private network, sorted — the same set
|
||||
// the names and the resolver mean by it (onTheNetwork), so a rule saying "from the mesh" admits
|
||||
// exactly the machines the mesh names.
|
||||
//
|
||||
// A node with no address is left out rather than rendered as an empty source: an empty entry in a
|
||||
// source set is a syntax error in the rule file, and a rule file that does not load leaves the
|
||||
// node filtering whatever it was filtering before -- the one outcome worse than a wrong rule,
|
||||
// because nothing reports it.
|
||||
func onThePrivateNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) ([]string, error) {
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []string
|
||||
for _, p := range places {
|
||||
if strings.TrimSpace(p.Address) != "" {
|
||||
out = append(out, p.Address)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// namesInTheMesh is every machine's internal name and the address behind it — every machine
|
||||
// that is on the private network, the same set the resolver means by that.
|
||||
//
|
||||
// A machine that is not has no name: writing one that resolves to nothing is worse than not
|
||||
// writing it, because a connection to an address that does not answer hangs where a name that
|
||||
// does not resolve fails at once and says so. A machine placed on the overlay but not running
|
||||
// the module that puts it there is exactly that (novox/hq issue 079).
|
||||
func namesInTheMesh(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest) (map[string]string, error) {
|
||||
places, err := onTheNetwork(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, p := range places {
|
||||
out[overlay.InternalName(p.Name)] = p.Address
|
||||
}
|
||||
// And the carried peers the operator has named (novox/hq issue 112): machines the
|
||||
// predecessor's resolver answers for and the mesh routes to, known by name on the operator's
|
||||
// word until they enrol — at which point enrolment verifies the name and the node's own
|
||||
// entry takes over above. A name the predecessor answers for must keep resolving until the
|
||||
// machine behind it is a node; without these, taking the resolver silences three machines.
|
||||
carried, err := inv.CarriedPeers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range carried {
|
||||
if p.Named == "" || p.EnrolledAs != "" {
|
||||
continue
|
||||
}
|
||||
if _, taken := out[overlay.InternalName(p.Named)]; taken {
|
||||
continue // a node of the mesh owns the name; the stale statement loses
|
||||
}
|
||||
out[overlay.InternalName(p.Named)] = p.Address
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// artifactStoreOnNetwork is the machine on this network that offers the artifact store, and the
|
||||
// port THAT MACHINE put it on — the node's setting when it was given one (novox/hq ADR 0100,
|
||||
// 04-ISSUES/102), the mesh's assignment when it made one, and the manifest's own number only when
|
||||
// neither says anything. Read exactly as a consumer's binding is, because the trust a machine
|
||||
// writes for the store and the address it pulls from are the same fact as what a consumer is told.
|
||||
//
|
||||
// A lookup failure is an error, never "not found": collapsing the two composed a declaration
|
||||
// without the trust whenever the inventory hiccuped, delivered by a push that reported success —
|
||||
// and nothing recomposed the machine until the next push. "No store" must mean the mesh has none,
|
||||
// not that the question went unanswered.
|
||||
func artifactStoreOnNetwork(ctx context.Context, inv *inventory.Inventory,
|
||||
on map[string]bool) (node, port string, found bool, err error) {
|
||||
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading the catalogue: %w", err)
|
||||
}
|
||||
providers := map[string]catalogue.Manifest{}
|
||||
for name, m := range shelf {
|
||||
if _, offers := m.Serves[catalogue.ArtifactStoreProvision]; offers {
|
||||
providers[name] = m
|
||||
}
|
||||
}
|
||||
if len(providers) == 0 {
|
||||
return "", "", false, nil
|
||||
}
|
||||
// In a stated order, so two machines offering it would always answer the same one.
|
||||
machines := make([]string, 0, len(on))
|
||||
for machine := range on {
|
||||
machines = append(machines, machine)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
for _, machine := range machines {
|
||||
assigned, err := inv.Assigned(ctx, machine)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading what %s is assigned: %w", machine, err)
|
||||
}
|
||||
for _, a := range assigned {
|
||||
m, offers := providers[a]
|
||||
if !offers {
|
||||
continue
|
||||
}
|
||||
serves, err := servedOnNode(ctx, inv, machine, m, catalogue.ArtifactStoreProvision)
|
||||
if err != nil {
|
||||
return "", "", false, fmt.Errorf("reading where %s puts the artifact store: %w", machine, err)
|
||||
}
|
||||
if p, ok := serves["port"]; ok {
|
||||
return machine, fmt.Sprintf("%v", p), true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", "", false, nil
|
||||
}
|
||||
|
||||
// artifactStoreAddress is the artifact store as `forNode` reaches it: `<node>.internal:<port>`
|
||||
// over the private network, or — when nothing is on the network yet — `127.0.0.1:<port>` for the
|
||||
// node that holds the store itself, and "" for any other. The address composed into every
|
||||
// reference the mesh built, at the moment it is used and never before (novox/hq 04-ISSUES/102).
|
||||
//
|
||||
// **Genesis places the network after the store, the broker, the vault and the catalogue.** Each
|
||||
// of those is built and pushed to a node that is on no network, and the store is on that same
|
||||
// node; an answer of "no store" there would refuse every one of those pushes and hand every one
|
||||
// of those builds a base nothing can pull. Loopback is the truth on that machine, and it is the
|
||||
// address genesis itself reaches the store by.
|
||||
func artifactStoreAddress(ctx context.Context, inv *inventory.Inventory,
|
||||
shelf map[string]catalogue.Manifest, forNode string) (string, error) {
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
on := map[string]bool{}
|
||||
for name := range onNetwork {
|
||||
on[name] = true
|
||||
}
|
||||
node, port, found, err := artifactStoreOnNetwork(ctx, inv, on)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if found {
|
||||
return overlay.InternalName(node) + ":" + port, nil
|
||||
}
|
||||
holder, port, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found || holder != forNode {
|
||||
return "", err
|
||||
}
|
||||
return "127.0.0.1:" + port, nil
|
||||
}
|
||||
|
||||
// artifactStoreHolder is whichever node is assigned a module offering the artifact store, on or
|
||||
// off the network, and the port that node put it on.
|
||||
func artifactStoreHolder(ctx context.Context, inv *inventory.Inventory) (node, port string, found bool, err error) {
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
all := map[string]bool{}
|
||||
for _, n := range nodes {
|
||||
all[n.Name] = true
|
||||
}
|
||||
return artifactStoreOnNetwork(ctx, inv, all)
|
||||
}
|
||||
@@ -0,0 +1,337 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// placementOf is what the mesh holds about where one node is.
|
||||
func placementOf(t *testing.T, ctx context.Context, inv *inventory.Inventory, name string) inventory.Overlay {
|
||||
t.Helper()
|
||||
placed, err := inv.Overlays(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, one := range placed {
|
||||
if one.Name == name {
|
||||
return one
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s is not placed at all", name)
|
||||
return inventory.Overlay{}
|
||||
}
|
||||
|
||||
// The sibling of `node public-domain`, and the worse one: a placement is three facts declared
|
||||
// together, so an invocation that said none of them took all three away — the endpoint every other
|
||||
// machine dials, the site, and the hub. A mesh whose hub was placed that way has no paths left.
|
||||
func TestPlacingANodeWithNothingSaidDoesNotUnplaceIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := overlayPlace(ctx, open.inventory,
|
||||
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err := overlayPlace(ctx, open.inventory, []string{"anchor"})
|
||||
if err == nil {
|
||||
t.Fatal("saying nothing unplaced the node instead of being refused")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "--nothing") {
|
||||
t.Errorf("the refusal does not say how to mean it: %v", err)
|
||||
}
|
||||
|
||||
held := placementOf(t, ctx, open.inventory, "anchor")
|
||||
if held.Endpoint != "198.51.100.10:51820" || held.Site != "hosting" || !held.Hub {
|
||||
t.Fatalf("the placement was taken away by an invocation that was refused: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
// Placing a machine with nothing set is a real thing to want — one that roams and opens every path
|
||||
// itself is exactly that — so it keeps a way to be said, by name.
|
||||
func TestPlacingANodeWithNothingIsAskedForByName(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := overlayPlace(ctx, open.inventory,
|
||||
[]string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, open.inventory, []string{"anchor", "--nothing"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := placementOf(t, ctx, open.inventory, "anchor")
|
||||
if held.Endpoint != "" || held.Site != "" || held.Hub {
|
||||
t.Fatalf("--nothing did not place it with nothing: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
// Both at once cannot be meant, so neither silently wins.
|
||||
func TestAPlacementAndNothingTogetherIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := overlayPlace(ctx, open.inventory,
|
||||
[]string{"anchor", "--site", "hosting", "--nothing"}); err == nil {
|
||||
t.Fatal("a placement and --nothing together was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine is named for the others only while it is on the private network — placed AND running
|
||||
// what puts it there — the same set the resolver means by "on the private network". A machine
|
||||
// that has an address and no networking is not named: a name resolving to an address that does
|
||||
// not answer hangs where an unknown name fails at once (novox/hq issue 079).
|
||||
func TestOnlyAMachineOnThePrivateNetworkIsNamed(t *testing.T) {
|
||||
open := aMesh(t) // two placed machines, both assigned what puts them on the private network
|
||||
ctx := t.Context()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, err := namesInTheMesh(ctx, open.inventory, shelf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if names["anchor.internal"] != "10.77.0.1" || names["laptop.internal"] != "10.77.0.2" {
|
||||
t.Fatalf("two machines on the network are not both named: %v", names)
|
||||
}
|
||||
// The laptop keeps its place and its address, and stops running the network.
|
||||
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names, err = namesInTheMesh(ctx, open.inventory, shelf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := names["laptop.internal"]; still || names["anchor.internal"] != "10.77.0.1" {
|
||||
t.Fatalf("a machine that left the network is still named, or the one that stayed is not: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0105: the range every address is composed from is the adopted tunnel's, read from
|
||||
// the tunnel the hub holds — never stored anywhere else.
|
||||
func TestTheOverlaysRangeIsTheAdoptedTunnels(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
t.Setenv(OverlayCIDRVar, "10.99.0.0/16")
|
||||
|
||||
before, err := overlayRange(ctx, inv)
|
||||
if err != nil || before != "10.99.0.0/16" {
|
||||
t.Fatalf("without an adopted tunnel the range is not what genesis said: %q %v", before, err)
|
||||
}
|
||||
|
||||
// The hub becomes what genesis makes of a machine in use: adopted, enrolled with the found
|
||||
// tunnel's key, and presenting the tunnel.
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key,
|
||||
Peers: []inventory.TunnelPeer{{PublicKey: "PEER-TWO", Address: "192.0.2.2"}},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
after, err := overlayRange(ctx, inv)
|
||||
if err != nil || after != "192.0.2.0/24" {
|
||||
t.Fatalf("with an adopted tunnel the range is %q (%v), not the tunnel's", after, err)
|
||||
}
|
||||
|
||||
// A placement whose endpoint is on another port than the tunnel's is refused: the peers dial
|
||||
// the tunnel's port.
|
||||
err = overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51820", "--site", "hosting", "--hub"})
|
||||
if err == nil || !strings.Contains(err.Error(), "51900") {
|
||||
t.Fatalf("an endpoint off the tunnel's port was accepted: %v", err)
|
||||
}
|
||||
// On the tunnel's port, the hub is placed at the tunnel's address — whatever it had before.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "hosting", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if placed := placementOf(t, ctx, inv, "anchor"); placed.Address != "192.0.2.1" {
|
||||
t.Fatalf("the hub was placed at %s, not the tunnel's own address", placed.Address)
|
||||
}
|
||||
|
||||
// And the hub's declaration carries the peer and the takeover.
|
||||
nodes, computed, err := graph(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var hubNode overlay.Node
|
||||
for _, n := range nodes {
|
||||
if n.Name == "anchor" {
|
||||
hubNode = n
|
||||
}
|
||||
}
|
||||
if hubNode.TakesOver == nil || hubNode.TakesOver.Unit != "wg-quick@wg0" {
|
||||
t.Errorf("the hub is not told to take over the found tunnel: %+v", hubNode)
|
||||
}
|
||||
carried := false
|
||||
for _, p := range computed["anchor"] {
|
||||
if p.Key == "PEER-TWO" && p.Allowed == "192.0.2.2/32" {
|
||||
carried = true
|
||||
}
|
||||
}
|
||||
if !carried {
|
||||
t.Errorf("the hub's peer list does not carry the tunnel's peer: %+v", computed["anchor"])
|
||||
}
|
||||
}
|
||||
|
||||
// A takeover is composed only for a hub whose placement agrees with the tunnel: an address or an
|
||||
// endpoint port that differs would have the host stop the found interface and raise the mesh's
|
||||
// where no peer is listening.
|
||||
func TestATakeoverIsNotComposedForAHubPlacedOffItsTunnel(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if err := inv.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hub, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "THE-TUNNELS-KEY========================="
|
||||
if err := inv.RecordOverlayKey(ctx, hub.ID, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordTunnel(ctx, hub.ID, inventory.Tunnel{
|
||||
Interface: "wg0", Unit: "wg-quick@wg0", Config: "/etc/wireguard/wg0.conf", Port: 51900,
|
||||
Address: "192.0.2.1/24", Range: "192.0.2.0/24", PublicKey: key}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// aMesh placed anchor at 10.77.0.1 on :51820 — the record of a hub placed before it took the
|
||||
// tunnel over.
|
||||
_, _, err = graph(ctx, open)
|
||||
if err == nil {
|
||||
t.Fatal("a takeover was composed for a hub whose address and port are not the tunnel's")
|
||||
}
|
||||
for _, want := range []string{"10.77.0.1", "192.0.2.1", "51820", "51900", "overlay place anchor"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
// Re-placed on the tunnel, it composes.
|
||||
if err := inv.SetPlace(ctx, "anchor", "", "", false, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := overlayPlace(ctx, inv, []string{"anchor", "--endpoint", "198.51.100.10:51900", "--site", "here", "--hub"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := graph(ctx, open); err != nil {
|
||||
t.Fatalf("re-placed on the tunnel, the graph still refuses: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// theResolver is the catalogue's dnsmasq module as it is, or the test is skipped where the
|
||||
// catalogue is not beside this checkout.
|
||||
func theResolver(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("../../../mesh-catalog/modules/dnsmasq/module.json")
|
||||
if err != nil {
|
||||
t.Skipf("the catalogue is not beside this checkout: %v", err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("dnsmasq does not parse:\n%v", err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// The resolver is handed every machine on the private network as a wildcard, and is handed it again
|
||||
// when a machine leaves — through the module's own manifest asking for the fact, with no module of the
|
||||
// mesh's own in between (hal dnsmasq-app conversion, novox/hq 08-connectivity). It is the mesh's one
|
||||
// resolver (ADR 0194), and the container runtime is given no resolver of its own (ADR 0196).
|
||||
func TestTheResolverIsToldEveryMachineOnTheNetworkAndToldAgainWhenOneLeaves(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, theResolver(t))
|
||||
if _, err := assign(ctx, open, "anchor", "dnsmasq"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its bus credential, as assigning issues it where the bus is reachable (novox/hq issue 203):
|
||||
// no bus is known to this test, so it is minted here, or composing refuses the placeholder.
|
||||
if _, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "anchor.dnsmasq", Kind: inventory.BusModule, Node: "anchor", Module: "dnsmasq"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zones := func() string {
|
||||
t.Helper()
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.fact-node-zones" {
|
||||
if r["path"] != "/etc/mesh-resolver/nodes.conf" {
|
||||
t.Fatalf("the machines were written somewhere the resolver does not read: %v", r["path"])
|
||||
}
|
||||
return r["content"].(string)
|
||||
}
|
||||
}
|
||||
t.Fatal("the resolver was not handed the machines")
|
||||
return ""
|
||||
}
|
||||
first := zones()
|
||||
for _, want := range []string{
|
||||
"local=/internal/", "address=/anchor.internal/10.77.0.1\n", "address=/laptop.internal/10.77.0.2\n",
|
||||
} {
|
||||
if !strings.Contains(first, want) {
|
||||
t.Errorf("the resolver's machines lack %q:\n%s", want, first)
|
||||
}
|
||||
}
|
||||
// The container runtime is given no resolver of its own (novox/hq ADR 0196): it copies its
|
||||
// machine's, which name the mesh's resolver first. A `dns` key would be a second account of where a
|
||||
// container asks, read only when the runtime starts.
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "dnsmasq.runtime-dns" {
|
||||
t.Errorf("the resolver still writes the runtime's own dns: %v", r)
|
||||
}
|
||||
}
|
||||
|
||||
// The laptop keeps its place and its address, and stops running the network.
|
||||
if err := open.inventory.Unassign(ctx, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after := zones()
|
||||
if strings.Contains(after, "laptop") || !strings.Contains(after, "address=/anchor.internal/10.77.0.1\n") {
|
||||
t.Fatalf("a machine that left the network is still a wildcard, or the one that stayed is not:\n%s", after)
|
||||
}
|
||||
}
|
||||
|
||||
// The roster is the machines and nothing else (novox/hq ADR 0191): each node's internal domain covers
|
||||
// every route on it, and a node's public domains are public DNS's. A routed name in `.Names` was a
|
||||
// private answer for a public name, handed by a resolver serving a LAN to a phone that could not use it.
|
||||
func TestTheRosterNamesOnlyTheMachines(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
with, _, err := renderingFor(ctx, open, node, plan, settings, gens, Reading)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(with.Names, with.Machines) {
|
||||
t.Fatalf("%s's roster names more than the machines:\n names %v\n machines %v", node, with.Names, with.Machines)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/token"
|
||||
)
|
||||
|
||||
// what a machine is, and what it is allowed to be told.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
func nodeCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New("node add <name>, node list, node show <name>, or " + publicDomainUsage)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
switch args[0] {
|
||||
case "show":
|
||||
if len(args) != 2 {
|
||||
return errors.New("node show <name>")
|
||||
}
|
||||
return showNode(ctx, inv, args[1])
|
||||
case "add":
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
case "list":
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// **The same list, for something other than a person** — the console's discovery reads it
|
||||
// (novox/hq ADR 0195), and a reader that parses a printed column breaks when it is reworded.
|
||||
if len(args) > 1 && args[1] == "--json" {
|
||||
type listed struct {
|
||||
Name string `json:"name"`
|
||||
Heard string `json:"heard"`
|
||||
Mode string `json:"mode"`
|
||||
ID string `json:"id"`
|
||||
}
|
||||
out := make([]listed, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
out = append(out, listed{Name: n.Name, Heard: heardFrom(n), Mode: modeOf(n), ID: n.ID})
|
||||
}
|
||||
return printJSON(out)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// Said rather than printed as nothing: an empty list and a failed read must never
|
||||
// look the same, and this command answering "none" is only honest because getting
|
||||
// here means the store answered.
|
||||
fmt.Println("this mesh has no node records yet")
|
||||
return nil
|
||||
}
|
||||
for _, n := range nodes {
|
||||
fmt.Printf("%-20s %-14s %-9s %s\n", n.Name, heardFrom(n), modeOf(n), n.ID)
|
||||
}
|
||||
return nil
|
||||
|
||||
case "public-domain":
|
||||
// The domain this node composes its routed names under (novox/hq ADR 0066).
|
||||
//
|
||||
// **The form with no argument reports; clearing is asked for by name.** It used to clear —
|
||||
// so `node public-domain anchor`, which reads like a question and is what anybody types to
|
||||
// find out what the answer is, silently took every routed name the node had. A read-shaped
|
||||
// invocation must never be a destructive write: there is no output that makes up for it,
|
||||
// because the damage is already done by the time it prints.
|
||||
return publicDomain(ctx, inv, args[1:])
|
||||
|
||||
case "networks":
|
||||
// Removed by novox/hq ADR 0140, which superseded the record that added it. The filter no
|
||||
// longer names any network: it constrains what arrives from outside the machine and says
|
||||
// nothing about what did not, so there is no list to keep. Answered rather than met with
|
||||
// "unknown command", because this was the documented way to stop a flip cutting a machine's
|
||||
// containers off and somebody will reasonably still type it.
|
||||
return errors.New("`node networks` is gone (novox/hq ADR 0140). The filter constrains what " +
|
||||
"arrives from outside this machine and says nothing about traffic that did not, so no " +
|
||||
"network is named anywhere and nothing needs to be said to keep a machine's own " +
|
||||
"containers reaching outward. The machine reports which of its links face outside; see " +
|
||||
"`node show <name>`")
|
||||
|
||||
case "account":
|
||||
// The operator's login on this machine (novox/hq to-be 29): what a home-scoped file is
|
||||
// owned by and which account `ssh <node>` uses. Reports with no argument; sets with one;
|
||||
// an optional second argument is the home when it is not /home/<account>.
|
||||
return nodeAccount(ctx, inv, args[1:])
|
||||
|
||||
default:
|
||||
return fmt.Errorf("node has no %q; it has add, list, show, public-domain and account", args[0])
|
||||
}
|
||||
}
|
||||
|
||||
// addNode creates a node record, adopted when the operator says so (novox/hq ADR 0100).
|
||||
func addNode(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node add", flag.ContinueOnError)
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine is in use: keep what is found on it until each module is taken")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("node add <name> [--adopted]")
|
||||
}
|
||||
node, err := inv.AddNodeAs(ctx, positionals[0], *adopted)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("added %s (%s)", node.Name, node.ID)
|
||||
if node.Adopted {
|
||||
fmt.Print(", adopted")
|
||||
}
|
||||
fmt.Println()
|
||||
return nil
|
||||
}
|
||||
|
||||
// modeOf is a node's mode as a word (novox/hq ADR 0100): an adopted node is said to be adopted
|
||||
// wherever the mesh reports a node's state.
|
||||
func modeOf(n inventory.Node) string {
|
||||
if n.Adopted {
|
||||
return "adopted"
|
||||
}
|
||||
return "converged"
|
||||
}
|
||||
|
||||
// publicDomainUsage is the one description of the three forms, so a refusal and the help agree.
|
||||
// nodeAccount reports or sets a node's operator account (novox/hq to-be 29). Read-shaped with no
|
||||
// argument, like public-domain: `node account novox` answers, it does not change anything.
|
||||
func nodeAccount(ctx context.Context, inv *inventory.Inventory, positionals []string) error {
|
||||
if len(positionals) == 0 || len(positionals) > 3 {
|
||||
return errors.New("node account <name> — what it is now; " +
|
||||
"node account <name> <account> [home] — set it (home defaults to /home/<account>)")
|
||||
}
|
||||
node := positionals[0]
|
||||
if len(positionals) == 1 {
|
||||
who, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if who.Account == "" {
|
||||
fmt.Printf("%s has no operator account known\n", node)
|
||||
fmt.Printf(" `node account %s <account>` sets it\n", node)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s logs a person in as %s (home %s)\n", node, who.Account, who.Home())
|
||||
return nil
|
||||
}
|
||||
home := ""
|
||||
if len(positionals) == 3 {
|
||||
home = positionals[2]
|
||||
}
|
||||
if err := inv.SetAccount(ctx, node, positionals[1], home); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s logs a person in as %s\n", node, positionals[1])
|
||||
fmt.Printf(" run `push %s` once ssh-client is assigned, to send its operator config\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const publicDomainUsage = "node public-domain <name> — what it is now; " +
|
||||
"<name> <domain> to set it; <name> --clear to take it away"
|
||||
|
||||
// publicDomain reads, sets or clears the domain a node composes its routed names under.
|
||||
//
|
||||
// Three forms, and the destructive one is the only one that has to be asked for. Clearing is a
|
||||
// real thing to want — a machine that stops facing the outside composes no names, and
|
||||
// lab-versus-production is this one setting (novox/hq ADR 0066) — so it keeps a way to say it.
|
||||
// What it does not keep is being the thing that happens when nothing was said at all.
|
||||
func publicDomain(ctx context.Context, inv *inventory.Inventory, args []string) error {
|
||||
set := flag.NewFlagSet("node public-domain", flag.ContinueOnError)
|
||||
clear := set.Bool("clear", false, "take the domain away; it composes no routed names after")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 || len(positionals) > 2 {
|
||||
return errors.New(publicDomainUsage)
|
||||
}
|
||||
node := positionals[0]
|
||||
|
||||
switch {
|
||||
case *clear && len(positionals) == 2:
|
||||
// Both, which cannot be meant. Refused rather than one of them silently winning.
|
||||
return fmt.Errorf("give %s a domain or --clear, not both: %q and --clear say opposite "+
|
||||
"things and the mesh will not choose between them", node, positionals[1])
|
||||
|
||||
case *clear:
|
||||
if err := inv.SetPublicDomain(ctx, node, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
|
||||
fmt.Printf(" run `push %s` to take them off it\n", node)
|
||||
return nil
|
||||
|
||||
case len(positionals) == 2:
|
||||
if err := inv.SetPublicDomain(ctx, node, positionals[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s composes its routed names under %s\n", node, positionals[1])
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
|
||||
default:
|
||||
// Asked, so answered. NodeByName first, so a name the mesh has never heard of is a refusal
|
||||
// rather than "it has no public domain", which is true of that name and says nothing.
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
domain, err := inv.PublicDomainOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if domain == "" {
|
||||
fmt.Printf("%s has no public domain, so it composes no routed names\n", node)
|
||||
fmt.Printf(" `node public-domain %s <domain>` gives it one\n", node)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s composes its routed names under %s\n", node, domain)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func tokenCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 || args[0] != "issue" {
|
||||
return errors.New("token issue --node <name>, or token issue --new <name>")
|
||||
}
|
||||
|
||||
set := flag.NewFlagSet("token issue", flag.ContinueOnError)
|
||||
existing := set.String("node", "", "issue for a node record that already exists")
|
||||
fresh := set.String("new", "", "create the node record, then issue for it")
|
||||
validFor := set.Duration("for", time.Hour, "how long the token may be used")
|
||||
adopted := set.Bool("adopted", false,
|
||||
"the machine joining is in use: it is adopted, and keeps what is found on it")
|
||||
if err := set.Parse(args[1:]); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Exactly one, because the difference is what the token binds to. A command that guessed
|
||||
// would sometimes create a second record for a machine that already has one.
|
||||
if (*existing == "") == (*fresh == "") {
|
||||
return errors.New("give exactly one of --node <name> or --new <name>: the first is a " +
|
||||
"machine the mesh already has a record for, the second is one it has never seen")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
issued, err := issueFor(ctx, inv, *existing, *fresh, *adopted, *validFor)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Assembled from two contexts by the process that holds both grants. Neither reads the
|
||||
// other's store (novox/hq ADR 0008) — each is asked for its own part.
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
key, err := ident.Establish(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The account is created before the token is handed over, which is what removes the
|
||||
// chicken-and-egg entirely: the mesh runs the broker, so a joining node's credentials can
|
||||
// exist before it does. The one-time secret IS the password, so a node's first connection is
|
||||
// already authenticated and enrolment is what happens over it.
|
||||
|
||||
made := token.Token{Node: issued.Node.Name, Signer: key.Public, Secret: issued.Secret,
|
||||
Adopted: issued.Node.Adopted}
|
||||
|
||||
// Absent is a state, not a failure: a control plane can hold records and a key before it has
|
||||
// a broker. What it cannot do is issue a token anybody could use, and Missing() says so.
|
||||
known, err := broker.FromEnvironment()
|
||||
switch {
|
||||
case err == nil:
|
||||
made.Broker, made.Fingerprint = known.Address, known.Fingerprint
|
||||
case errors.Is(err, broker.ErrNotConfigured):
|
||||
default:
|
||||
return err
|
||||
}
|
||||
encoded, err := made.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
joins := ""
|
||||
if made.Adopted {
|
||||
joins = ", joining adopted"
|
||||
}
|
||||
fmt.Printf("token for %s%s, usable once, until %s\n\n %s\n\n",
|
||||
issued.Node.Name, joins, issued.Expires.Format(time.RFC3339), encoded)
|
||||
fmt.Println("This is the only time it is shown. What is stored is a hash of the secret.")
|
||||
|
||||
if missing := made.Missing(); len(missing) > 0 {
|
||||
fmt.Printf("\nINCOMPLETE — this token cannot be used to join anything yet. Missing:\n")
|
||||
for _, m := range missing {
|
||||
fmt.Printf(" - %s\n", m)
|
||||
}
|
||||
fmt.Printf("\nSet %s and %s once the broker is raised.\n",
|
||||
broker.AddressVar, broker.CertificateVar)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// issueFor is the inventory's half of issuing a token: the record, made when it is new, adopted
|
||||
// when the operator says so, and the one-time secret for it. The node in what it returns carries
|
||||
// its mode, which is what the token says.
|
||||
func issueFor(ctx context.Context, inv *inventory.Inventory, existing, fresh string, adopted bool,
|
||||
validFor time.Duration) (inventory.Issued, error) {
|
||||
name := existing
|
||||
if fresh != "" {
|
||||
node, err := inv.AddNodeAs(ctx, fresh, adopted)
|
||||
if err != nil {
|
||||
return inventory.Issued{}, err
|
||||
}
|
||||
name = node.Name
|
||||
}
|
||||
// Not saying adopted leaves the node as it is: re-issuing a token for an adopted node does not
|
||||
// converge it — converging is its own act, previewed (novox/hq ADR 0100). And saying it for a
|
||||
// node already converged is refused rather than done quietly: returning a node to adopted is
|
||||
// its own act too, which unloads the mesh's filter and enables the found firewall again.
|
||||
if adopted && fresh == "" {
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return inventory.Issued{}, err
|
||||
}
|
||||
if !node.Adopted {
|
||||
return inventory.Issued{}, fmt.Errorf("%s is converged, and a token does not change "+
|
||||
"that: run `adopt %s` to return it to adopted, then issue the token without "+
|
||||
"--adopted", name, name)
|
||||
}
|
||||
}
|
||||
|
||||
return inv.IssueToken(ctx, name, validFor)
|
||||
}
|
||||
|
||||
func identityCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("identity show")
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
// Establish rather than read: a control plane asked for its identity before it has one should
|
||||
// get one, not an error. Generating it is idempotent, so this is safe to run at any time.
|
||||
key, err := ident.Establish(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("signing key %s\n", key.ID)
|
||||
fmt.Printf("fingerprint %s\n", key.Fingerprint())
|
||||
fmt.Printf("created %s\n", key.Created.Format(time.RFC3339))
|
||||
fmt.Printf("\nThe public half of this travels in every enrolment token. A node believes a\n" +
|
||||
"declaration because it carries a signature this key made (novox/hq ADR 0004).\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func brokerCommand(ctx context.Context, args []string) error {
|
||||
if len(args) > 0 && args[0] == "certificate" {
|
||||
return busCertificate(args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "accounts" {
|
||||
return busAccounts(ctx, args[1:])
|
||||
}
|
||||
if len(args) > 0 && args[0] == "consumer-reset" {
|
||||
return consumerReset(args[1:])
|
||||
}
|
||||
if len(args) == 0 || args[0] != "show" {
|
||||
return errors.New("broker show | broker certificate [--check] --into <directory> | broker accounts --into <file> | " +
|
||||
"broker consumer-reset <stream> <consumer>")
|
||||
}
|
||||
known, err := broker.FromEnvironment()
|
||||
if errors.Is(err, broker.ErrNotConfigured) {
|
||||
fmt.Printf("no broker configured. Set %s and %s.\n\n"+
|
||||
"Until then tokens carry the signing key and the one-time secret, and say what they\n"+
|
||||
"are missing. They cannot be used to join.\n",
|
||||
broker.AddressVar, broker.CertificateVar)
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("address %s\n", known.Address)
|
||||
fmt.Printf("fingerprint %s\n", known.Fingerprint)
|
||||
fmt.Print("\nThe fingerprint is computed from the certificate on disk, never configured. A\n" +
|
||||
"node checks it before sending anything (novox/hq ADR 0004).\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumerReset re-makes one consumer on a stream that keeps history to start from now (novox/hq issue
|
||||
// 248): the way out of a consumer replaying a week of announcements, said rather than done by hand. A
|
||||
// person's act — what was pending is dropped — so it is a command, and nothing calls it on its own.
|
||||
func consumerReset(args []string) error {
|
||||
if len(args) != 2 {
|
||||
return errors.New("broker consumer-reset <stream> <consumer>, e.g. broker consumer-reset EVENTS controller")
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
defer js.Close()
|
||||
before, after, err := js.ResetConsumer(args[0], args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("consumer %s on %s re-made to deliver from now\n", args[1], args[0])
|
||||
fmt.Printf(" before: delivers %s, delivered to %d, acknowledged to %d, %d pending, %d unacknowledged\n",
|
||||
before.DeliverPolicy, before.Delivered, before.AckFloor, before.Pending, before.AckPending)
|
||||
fmt.Printf(" after: delivers %s, %d pending; what was pending is dropped. A holder bound to it may need its "+
|
||||
"process restarted to bind again\n", after.DeliverPolicy, after.Pending)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
||||
// picture of the mesh.
|
||||
func heardFrom(n inventory.Node) string {
|
||||
silent, ever := n.Silent()
|
||||
switch {
|
||||
case !ever:
|
||||
return "never spoken"
|
||||
case silent > SilentFor:
|
||||
return "out of touch " + roughly(silent)
|
||||
default:
|
||||
return "here"
|
||||
}
|
||||
}
|
||||
|
||||
// roughly is a duration a person reads rather than parses.
|
||||
func roughly(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%dh", int(d.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
}
|
||||
}
|
||||
|
||||
// showNode says what one machine reported about itself, in its own words.
|
||||
//
|
||||
// **A capability is detected and never assumed** (novox/hq ADR 0009), so the only account of what
|
||||
// a machine can do is the one it gave — and its detail is half of that account. The mesh was
|
||||
// keeping the yes and discarding the reason, which makes *this machine has no seat* an answer with
|
||||
// nowhere to go: a person told a machine lacks something wants to know what the detector saw.
|
||||
//
|
||||
// It is also where "what should it be configured as" is read. The same line that gates an
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, inv *inventory.Inventory, name string) error {
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s\n", node.Name)
|
||||
fmt.Printf(" last heard from %s\n", heardFrom(node))
|
||||
// Which host runs it, as it reported (novox/hq 04-ISSUES/087). Said whenever known, because a
|
||||
// host refuses a declaration carrying a field it does not understand and refuses it WHOLE — so
|
||||
// which host a machine runs is what decides whether the mesh can send it anything new, and
|
||||
// nothing could say it. "not reported" rather than blank: a machine that has not said is a
|
||||
// different thing from one running nothing.
|
||||
fmt.Printf(" host %s\n", orNotReported(node.HostVersion))
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The domain its routed names are composed under, when it has one (novox/hq ADR 0066). Shown
|
||||
// only when set: a machine that serves nothing to the outside has no domain, and saying so of
|
||||
// every internal node would be noise.
|
||||
domain, err := inv.PublicDomainOf(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if domain != "" {
|
||||
fmt.Printf(" public domain %s\n", domain)
|
||||
}
|
||||
|
||||
// A provider here failing a consumer, or a consumer here failed (novox/hq ADR 0224). Before the
|
||||
// capabilities, because it is something not working now and they are a description.
|
||||
failing, err := failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if here := failingOn(failing, name); len(here) > 0 {
|
||||
fmt.Printf("\n %d consumer(s) a provider keeps failing, here or for a module here:\n", len(here))
|
||||
for _, line := range failingLines(here, time.Now()) {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
|
||||
held, err := inv.Profile(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if held == nil {
|
||||
// Never reported is not the same as reported nothing, and the remedy differs: one is a
|
||||
// machine that has not run the host yet, the other is a machine that ran it and can do
|
||||
// nothing.
|
||||
fmt.Printf("\n this machine has never said what it can do, so everything requiring a\n" +
|
||||
" capability is refused here — run the host on it\n")
|
||||
return nil
|
||||
}
|
||||
if len(held) == 0 {
|
||||
fmt.Printf("\n it reported no capabilities at all\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("\n what it can do, as it reported:\n")
|
||||
for _, c := range held {
|
||||
mark := "no "
|
||||
if c.Present {
|
||||
mark = "yes"
|
||||
}
|
||||
fmt.Printf(" %s %-20s %s\n", mark, c.Name, c.Detail)
|
||||
}
|
||||
|
||||
assigned, err := inv.Assigned(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// orNotReported is a fact a machine states about itself, or the fact that it has not.
|
||||
func orNotReported(s string) string {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return "not reported — this machine has not said since the mesh began keeping it"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// printJSON prints a value as indented JSON, the shape every `--json` answers in.
|
||||
func printJSON(v any) error {
|
||||
body, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **A read-shaped invocation is never a destructive write.**
|
||||
//
|
||||
// `node public-domain anchor` used to clear the domain. It reads like a question — it is what
|
||||
// anybody types to find out what the answer is — and it silently took every routed name the node
|
||||
// had. There is no output that makes up for that: by the time it prints, the fact is gone.
|
||||
func TestAskingForANodesPublicDomainDoesNotTakeItAway(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := publicDomain(ctx, open.inventory, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if domain != "example.test" {
|
||||
t.Fatalf("asking what the domain is took it away: %q", domain)
|
||||
}
|
||||
}
|
||||
|
||||
// Clearing is a real thing to want — a machine that stops facing the outside composes no names —
|
||||
// so it keeps a way to be said. What it stops being is what happens when nothing was said.
|
||||
func TestClearingANodesPublicDomainIsAskedForByName(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := publicDomain(ctx, open.inventory, []string{"anchor", "--clear"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if domain != "" {
|
||||
t.Fatalf("--clear did not clear it: %q", domain)
|
||||
}
|
||||
}
|
||||
|
||||
// A domain sets it, as it always did.
|
||||
func TestGivingANodeAPublicDomainSetsIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := publicDomain(ctx, open.inventory, []string{"anchor", "example.test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if domain != "example.test" {
|
||||
t.Fatalf("got %q", domain)
|
||||
}
|
||||
}
|
||||
|
||||
// A domain and --clear say opposite things. Refused rather than one of them silently winning.
|
||||
func TestADomainAndClearTogetherIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := open.inventory.SetPublicDomain(ctx, "anchor", "example.test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := publicDomain(ctx, open.inventory, []string{"anchor", "other.test", "--clear"})
|
||||
if err == nil {
|
||||
t.Fatal("a domain and --clear together were accepted")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "not both") {
|
||||
t.Fatalf("the refusal does not say why: %v", err)
|
||||
}
|
||||
// And neither half happened.
|
||||
domain, err := open.inventory.PublicDomainOf(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if domain != "example.test" {
|
||||
t.Fatalf("a refused command changed something: %q", domain)
|
||||
}
|
||||
}
|
||||
|
||||
// Asking about a name the mesh has never heard of is a refusal, not "it has no public domain" —
|
||||
// which is true of that name and says nothing.
|
||||
func TestAskingAboutAMachineTheMeshHasNeverHeardOfIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
if err := publicDomain(t.Context(), open.inventory, []string{"nowhere"}); err == nil {
|
||||
t.Fatal("a name the mesh does not know was answered as if it were a machine")
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: the operator says a node is adopted — `node add --adopted` for a record, and
|
||||
// the token for a machine joining.
|
||||
func TestANodeAddedAdoptedIsAdopted(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if err := addNode(ctx, open.inventory, []string{"joiner", "--adopted"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
n, err := open.inventory.NodeByName(ctx, "joiner")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !n.Adopted {
|
||||
t.Fatal("node add --adopted made a converged node")
|
||||
}
|
||||
if err := addNode(ctx, open.inventory, []string{"plain"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "plain"); n.Adopted {
|
||||
t.Fatal("node add without --adopted made an adopted node")
|
||||
}
|
||||
}
|
||||
|
||||
func TestATokenIssuedAdoptedSaysSoAndReissuingDoesNotConverge(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
issued, err := issueFor(ctx, open.inventory, "", "joiner", true, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !issued.Node.Adopted {
|
||||
t.Fatal("a token issued --adopted is for a node that is not adopted")
|
||||
}
|
||||
again, err := issueFor(ctx, open.inventory, "joiner", "", false, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !again.Node.Adopted {
|
||||
t.Fatal("re-issuing without --adopted converged the node; converging is its own act")
|
||||
}
|
||||
// --adopted for a node already converged is refused, and points at the act that does it.
|
||||
if _, err := issueFor(ctx, open.inventory, "laptop", "", true, time.Hour); err == nil ||
|
||||
!strings.Contains(err.Error(), "adopt laptop") {
|
||||
t.Fatalf("--adopted on a converged node was not refused: %v", err)
|
||||
}
|
||||
if n, _ := open.inventory.NodeByName(ctx, "laptop"); n.Adopted {
|
||||
t.Fatal("a refused token flipped the node to adopted")
|
||||
}
|
||||
// And said for a node that is adopted already, it is the ordinary re-issue.
|
||||
if _, err := issueFor(ctx, open.inventory, "joiner", "", true, time.Hour); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// operatorCommand is the mesh's one holder of secrets that is not a machine.
|
||||
//
|
||||
// **Every secret a module holds for itself is sealed to the node that uses it, and a node whose key
|
||||
// is gone takes its secrets with it** — the store's superuser and the broker's administrator among
|
||||
// them. novox/hq ADR 0085 (amended) gives them a second recipient: a person, with a sealing key
|
||||
// whose private half is made where the operator is and never enters the mesh. Making the key and
|
||||
// telling the mesh about it are two commands, on purpose: the first needs no mesh at all and runs
|
||||
// wherever the operator keeps things; the second gives the mesh the public half and nothing else.
|
||||
// The controller's own container is a scratch image with no writable path, which is the right
|
||||
// shape for a program that must hold no key — so the private half could not be written there
|
||||
// even by mistake.
|
||||
//
|
||||
// operator key make [--out <file>] make a keypair: private half to the file, public half printed
|
||||
// operator key set <public> tell the mesh which key to seal to
|
||||
// operator key show the public key, its fingerprint, and what it can recover
|
||||
const operatorUsage = "operator key make [--out <file>] | operator key set <public> [--replace] | " +
|
||||
"operator key show | operator issue <name> --invokes <tool,tool|*> | operator revoke <name> | " +
|
||||
"operator list"
|
||||
|
||||
func operatorCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
// The people who may reach the mesh's tools (design 25 §7). Beside the operator's key because
|
||||
// both answer "who, other than a machine, may do something here" — and a person reading this
|
||||
// command's usage is asking exactly that.
|
||||
switch args[0] {
|
||||
case "issue":
|
||||
return personIssue(ctx, args[1:])
|
||||
case "revoke":
|
||||
return personRevoke(ctx, args[1:])
|
||||
case "list":
|
||||
return personList(ctx)
|
||||
}
|
||||
if len(args) < 2 || args[0] != "key" {
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
switch args[1] {
|
||||
case "make":
|
||||
return operatorKeyMake(args[2:])
|
||||
case "set":
|
||||
return operatorKeySet(ctx, args[2:])
|
||||
case "show":
|
||||
return operatorKeyShow(ctx)
|
||||
default:
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
}
|
||||
|
||||
// operatorKeyMake needs no mesh: it is the operator's, run wherever the key will live.
|
||||
func operatorKeyMake(args []string) error {
|
||||
set := flag.NewFlagSet("operator key make", flag.ContinueOnError)
|
||||
out := set.String("out", "operator.key",
|
||||
"where to write the private key (0600); keep it off the mesh, and keep it")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
public, private, err := secrets.Keypair()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Create-exclusive: a key somebody may still need is never overwritten, and there is no window
|
||||
// between checking and writing in which one could appear.
|
||||
if err := writeNew(*out, []byte(private+"\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
|
||||
fmt.Printf(" private half written to %s (0600) — keep it off the mesh, and keep it\n", *out)
|
||||
fmt.Printf(" public half, to give the mesh with `operator key set`:\n")
|
||||
fmt.Printf("public %s\n", public)
|
||||
return nil
|
||||
}
|
||||
|
||||
func operatorKeySet(ctx context.Context, args []string) error {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("operator key set", flag.ContinueOnError)
|
||||
replace := set.Bool("replace", false,
|
||||
"replace an existing operator key — secrets sealed to the old one stay sealed to it")
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 1 {
|
||||
return errors.New(operatorUsage)
|
||||
}
|
||||
public := strings.TrimSpace(rest[0])
|
||||
if _, err := secrets.Seal(public, []byte("probe")); err != nil {
|
||||
return fmt.Errorf("that is not a public sealing key: %w", err)
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if current, err := inv.OperatorKey(ctx); err != nil {
|
||||
return err
|
||||
} else if current != "" && current != public && !*replace {
|
||||
return fmt.Errorf(
|
||||
"the mesh already has an operator key (%s). Pass --replace to change it — "+
|
||||
"secrets sealed to the current key stay sealed to it until each is issued again",
|
||||
secrets.Fingerprint(current))
|
||||
}
|
||||
orphaned, err := inv.SetOperatorKey(ctx, public)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("operator key %s\n", secrets.Fingerprint(public))
|
||||
fmt.Printf(" the mesh holds the public half only and cannot open what it seals to it;\n")
|
||||
fmt.Printf(" from now on every secret a module holds for itself is sealed to it as well.\n")
|
||||
fmt.Printf(" Secrets made before this cannot be — each is recoverable once issued again\n")
|
||||
if orphaned > 0 {
|
||||
fmt.Printf(" %d secret(s) are sealed to the previous key and stay so until issued again\n", orphaned)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func operatorKeyShow(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
key, err := inv.OperatorKey(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
fmt.Println("the mesh has no operator key; `operator key make` then `operator key set` gives it one")
|
||||
return nil
|
||||
}
|
||||
kept, earlier, unrecoverable, err := inv.KeptForOperator(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("operator key %s\n %s\n", secrets.Fingerprint(key), key)
|
||||
fmt.Printf(" %d secret(s) recoverable with it\n", len(kept))
|
||||
if len(earlier) > 0 {
|
||||
fmt.Printf(" %d secret(s) sealed to an earlier operator key — recoverable with that key only, until issued again:\n", len(earlier))
|
||||
for _, k := range earlier {
|
||||
fmt.Printf(" %s %s %s (%s)\n", k.Node, k.Module, k.Name, secrets.Fingerprint(k.Key))
|
||||
}
|
||||
}
|
||||
if len(unrecoverable) > 0 {
|
||||
fmt.Printf(" %d secret(s) not recoverable — made before the mesh had an operator key:\n", len(unrecoverable))
|
||||
for _, k := range unrecoverable {
|
||||
fmt.Printf(" %s %s %s\n", k.Node, k.Module, k.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readPrivateKey is the operator's key from the file `operator key make` wrote.
|
||||
func readPrivateKey(path string) (string, error) {
|
||||
if path == "" {
|
||||
return "", errors.New("--key <file> names the operator's private key, written by `operator key make`")
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(raw)), nil
|
||||
}
|
||||
|
||||
// personIssue gives somebody a credential for the mesh's tools, and prints it once.
|
||||
//
|
||||
// **Printed, not stored.** The mesh keeps a hash and nothing else, so this is the only moment the
|
||||
// credential exists anywhere but on the workstation that will use it — the same contract a token has,
|
||||
// and for the same reason: a credential recoverable from the mesh's store has the store's blast
|
||||
// radius.
|
||||
func personIssue(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("operator issue", flag.ContinueOnError)
|
||||
invokes := set.String("invokes", "", "the tools this person may call, comma-separated, or * for every one")
|
||||
// Flags on either side of the name, because the usage this command prints puts them after it —
|
||||
// and the standard parser stops at the first thing that is not a flag, so the order the command
|
||||
// documents was the one order it refused (2026-09-28).
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("operator issue <name> --invokes <tool,tool|*>")
|
||||
}
|
||||
name := positionals[0]
|
||||
if *invokes == "" {
|
||||
return errors.New(
|
||||
"say what this person may call: --invokes mesh-catalog.catalog_tools,gitea.repo_create, " +
|
||||
"or --invokes '*' for an administrator")
|
||||
}
|
||||
var tools []string
|
||||
for _, t := range strings.Split(*invokes, ",") {
|
||||
if t = strings.TrimSpace(t); t != "" {
|
||||
tools = append(tools, t)
|
||||
}
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if err := inv.RecordPerson(ctx, inventory.Person{Name: name, Invokes: tools}); err != nil {
|
||||
return err
|
||||
}
|
||||
// Refused here rather than at the next composition, where it would stop the whole file being
|
||||
// written for everybody. A name that cannot be part of a subject is one the server would read as
|
||||
// a wider permission than anybody granted.
|
||||
if _, err := broker.PermissionsFor(broker.Principal{
|
||||
Kind: broker.KindPerson, Module: name, Invokes: tools, PasswordHash: "x",
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
user := broker.Principal{Kind: broker.KindPerson, Module: name}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusPerson})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
where, err := broker.FromEnvironment()
|
||||
if err != nil && !errors.Is(err, broker.ErrNotConfigured) {
|
||||
return err
|
||||
}
|
||||
held, err := json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Person string `json:"person"`
|
||||
Invokes []string `json:"invokes"`
|
||||
}{
|
||||
URL: "nats://" + where.Address, Fingerprint: where.Fingerprint,
|
||||
User: user, Password: password, Person: name, Invokes: tools,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("issued %s, who may call %s\n", name, strings.Join(tools, ", "))
|
||||
fmt.Println(" this is the only time the credential is printed; the mesh keeps a hash")
|
||||
fmt.Println(" it works once the bus has been told, which is the next push to the machine holding mesh-broker")
|
||||
fmt.Println()
|
||||
fmt.Println(string(held))
|
||||
return nil
|
||||
}
|
||||
|
||||
func personRevoke(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("operator revoke <name>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
if err := open.inventory.ForgetPerson(ctx, args[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
// **Revoked at the next composition, not now.** The bus's users are a file, so a credential stops
|
||||
// working when the file no longer names it. Said plainly, because "revoked" that still works for
|
||||
// another minute is worth knowing about.
|
||||
fmt.Printf("%s is forgotten, and their credential stops working at the next composition — "+
|
||||
"push the machine holding mesh-broker to make it so\n", args[0])
|
||||
return nil
|
||||
}
|
||||
|
||||
func personList(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
people, err := open.inventory.People(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(people) == 0 {
|
||||
fmt.Println("nobody but machines reaches this mesh")
|
||||
return nil
|
||||
}
|
||||
for _, p := range people {
|
||||
fmt.Printf("%-20s %s\n", p.Name, strings.Join(p.Invokes, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// entry is a module as the catalogue holds it: built, so its manifest carries no `build` any more.
|
||||
func entry(module string, _ ...string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}}
|
||||
}
|
||||
|
||||
// stoodOn is what each module's newest build recorded it was handed.
|
||||
func stoodOn(edges map[string][]string) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for module, bases := range edges {
|
||||
for _, b := range bases {
|
||||
out[module] = append(out[module], catalogue.ArtifactStoreScheme+b+"/runtime@sha256:"+strings.Repeat("0", 64))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// A module built before the module it stands on is built against the old one and reports success
|
||||
// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived.
|
||||
func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) {
|
||||
in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")}
|
||||
edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}})
|
||||
got := orderByBases(in, edges)
|
||||
pos := map[string]int{}
|
||||
for i, e := range got {
|
||||
pos[e.Manifest.Module] = i
|
||||
}
|
||||
if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) {
|
||||
t.Fatalf("bases not first: %v", pos)
|
||||
}
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("an entry was lost or doubled: %d", len(got))
|
||||
}
|
||||
// A base outside the set is not waited for: it is not being rebuilt.
|
||||
got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}}))
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("a dependency outside the set changed the set: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A module registered from its manifest and never built still names its bases there; once built,
|
||||
// the recorded edge is what says so. Both are read, and a module never stands on itself.
|
||||
func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) {
|
||||
built := entry("gitea")
|
||||
edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}})
|
||||
if !standsOnModule(built, "mesh-tools", edges) {
|
||||
t.Fatal("a recorded edge was not read")
|
||||
}
|
||||
if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) {
|
||||
t.Fatal("an edge was invented")
|
||||
}
|
||||
fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{
|
||||
On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}},
|
||||
}}}
|
||||
if !standsOnModule(fresh, "mesh-tools", nil) {
|
||||
t.Fatal("a manifest's own base was not read")
|
||||
}
|
||||
}
|
||||
|
||||
// A merge names a repository the way the forge does; a source is recorded the way a build was
|
||||
// asked for. The two meet on owner/repo and branch, whichever form the record took.
|
||||
func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||
for _, s := range []inventory.Source{
|
||||
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"},
|
||||
{Repository: "novox/mesh-controller", Seat: "git", Ref: ""},
|
||||
{Repository: "https://elsewhere.example/novox/mesh-controller", Ref: "main"},
|
||||
} {
|
||||
if !sourceIs(s, m) {
|
||||
t.Errorf("%+v was not matched by the merge", s)
|
||||
}
|
||||
}
|
||||
for _, s := range []inventory.Source{
|
||||
{Repository: "novox/mesh-host", Seat: "git"},
|
||||
{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "release"},
|
||||
} {
|
||||
if sourceIs(s, m) {
|
||||
t.Errorf("%+v was matched by a merge that is not its", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A merge made before the source was last seen is history: it does not move the source, and a
|
||||
// merge that says nothing about when it was made is taken as news.
|
||||
func TestAMergeOlderThanTheLastLookIsHistory(t *testing.T) {
|
||||
seen := time.Date(2026, 9, 28, 3, 0, 0, 0, time.UTC)
|
||||
if !isHistory("2026-09-28T02:00:00Z", seen) {
|
||||
t.Fatal("an older merge was taken as news")
|
||||
}
|
||||
if isHistory("2026-09-28T04:00:00Z", seen) {
|
||||
t.Fatal("a newer merge was taken as history")
|
||||
}
|
||||
if isHistory("", seen) || isHistory("2026-09-28T02:00:00Z", time.Time{}) {
|
||||
t.Fatal("a merge or a source with no time on it was refused")
|
||||
}
|
||||
}
|
||||
|
||||
// A module as the catalogue holds it: built from a repository, at a directory inside it.
|
||||
func fromRepo(module, repository, path string) inventory.Entry {
|
||||
return inventory.Entry{
|
||||
Manifest: catalogue.Manifest{Module: module},
|
||||
Source: inventory.Source{Repository: repository, Path: path, Ref: "main"},
|
||||
}
|
||||
}
|
||||
|
||||
// A merge rebuilds the modules whose own directories it changed, and everything when what it changed
|
||||
// is shared. One repository holding many modules is the ordinary case here, and rebuilding all of
|
||||
// them for a change to one is what exhausted a registry's pull limit the first night this ran.
|
||||
func TestAMergeRebuildsTheModulesItChanged(t *testing.T) {
|
||||
const repo = "http://forge.internal:20000/novox/mesh-catalog.git"
|
||||
gitea := fromRepo("gitea", repo, "modules/gitea")
|
||||
keycloak := fromRepo("keycloak", repo, "modules/keycloak")
|
||||
known := []inventory.Entry{gitea, keycloak, fromRepo("plex", repo, "modules/plex")}
|
||||
candidates := []inventory.Entry{gitea, keycloak}
|
||||
merge := func(paths []string, truncated bool) link.SourceMoved {
|
||||
return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main",
|
||||
Paths: paths, PathsTruncated: truncated}
|
||||
}
|
||||
named := func(entries []inventory.Entry) string {
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Manifest.Module)
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
m link.SourceMoved
|
||||
want string
|
||||
}{
|
||||
{"one module's own files", merge([]string{"modules/gitea/index.ts", "modules/gitea/client.ts"}, false), "gitea"},
|
||||
{"two modules' files", merge([]string{"modules/gitea/index.ts", "modules/keycloak/module.json"}, false), "gitea,keycloak"},
|
||||
{"a file they share", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
{"a module the mesh does not hold", merge([]string{"modules/plex/index.ts"}, false), ""},
|
||||
{"nothing said about the files", merge(nil, false), "gitea,keycloak"},
|
||||
{"more files than were listed", merge([]string{"modules/gitea/index.ts"}, true), "gitea,keycloak"},
|
||||
// novox/hq issue 252: a module the mesh has never registered is still a module, when the merge
|
||||
// shows it is one — and a directory that may be shared code is still shared.
|
||||
{"a new module beside a held one", merge([]string{"modules/gitea/x", "modules/newmod/module.json"}, false), "gitea"},
|
||||
{"a new module's other files", merge([]string{"modules/newmod/index.ts", "modules/newmod/module.json"}, false), ""},
|
||||
{"a module removed", merge([]string{"modules/gone/module.json"}, false), ""},
|
||||
{"a directory with no manifest", merge([]string{"modules/lib/x.go"}, false), "gitea,keycloak"},
|
||||
{"a file directly among the modules", merge([]string{"modules/README.md"}, false), "gitea,keycloak"},
|
||||
{"the root's files still", merge([]string{"tsconfig.json"}, false), "gitea,keycloak"},
|
||||
} {
|
||||
if got := named(whatTheMergeTouched(candidates, known, c.m)); got != c.want {
|
||||
t.Errorf("%s: rebuilt %q, wanted %q", c.what, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A module whose recipe packages source from another repository is affected when that repository
|
||||
// moves — the manifest the mesh keeps says nothing about it, so the record of what the build read is
|
||||
// the only thing that can say so.
|
||||
func TestAModuleIsAffectedByTheRepositoryItPackages(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main",
|
||||
CloneURL: "http://forge.internal:20000/novox/mesh-controller.git"}
|
||||
for _, read := range [][]inventory.ReadRepository{
|
||||
{{Repository: "http://forge.internal:20000/novox/mesh-controller.git", Ref: "main"}},
|
||||
{{Repository: "novox/mesh-controller"}},
|
||||
{{Repository: "https://elsewhere.example/novox/other"}, {Repository: "novox/mesh-controller.git", Ref: "main"}},
|
||||
} {
|
||||
if !readsFrom(read, m) {
|
||||
t.Errorf("%+v was not matched by the merge", read)
|
||||
}
|
||||
}
|
||||
for _, read := range [][]inventory.ReadRepository{
|
||||
nil,
|
||||
{{Repository: "novox/mesh-host", Ref: "main"}},
|
||||
{{Repository: "novox/mesh-controller", Ref: "release"}},
|
||||
} {
|
||||
if readsFrom(read, m) {
|
||||
t.Errorf("%+v was matched by a merge that is not its", read)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// What a module handed over by hand records about where it came from, and what is refused.
|
||||
func TestWhatAHandedOverModuleRecordsAboutItsSource(t *testing.T) {
|
||||
// The whole location: a repository on the mesh's own forge, the directory inside it, the branch
|
||||
// and the commit the manifest was read at.
|
||||
from, err := whereItComesFrom("novox/mesh-catalog", "main", "c0ffee", "modules/gitea", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if from.Path != "modules/gitea" || from.Seat != "git" || from.Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("the source records as %+v", from)
|
||||
}
|
||||
// A manifest with no provenance at all is legitimate: fixing something in a hurry.
|
||||
if from, err := whereItComesFrom("", "", "", "", false); err != nil || !reflect.DeepEqual(from, inventory.Source{}) {
|
||||
t.Fatalf("a manifest handed over with no provenance was refused: %+v, %v", from, err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
repository, ref, commit, path string
|
||||
self bool
|
||||
}{
|
||||
{what: "a source with no commit", repository: "novox/mesh-catalog", commit: ""},
|
||||
{what: "a commit with no source", commit: "c0ffee"},
|
||||
{what: "a directory inside nothing", path: "modules/gitea"},
|
||||
{what: "a forge holding nothing", self: true},
|
||||
{what: "an address given as a path on the forge", repository: "http://forge.internal:20000/novox/x.git", commit: "c0ffee", self: true},
|
||||
} {
|
||||
if _, err := whereItComesFrom(c.repository, c.ref, c.commit, c.path, c.self); err == nil {
|
||||
t.Errorf("%s was recorded as a source", c.what)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/215: a module once built at a commit still follows its branch — a merge into it
|
||||
// matches the module, and a plan re-asks the branch, not the old commit.
|
||||
func TestAModuleBuiltAtACommitStillFollowsItsBranch(t *testing.T) {
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main"}
|
||||
pinned := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a"}
|
||||
if !sourceIs(pinned, m) {
|
||||
t.Error("a module whose record names a commit is left out of a merge into its branch")
|
||||
}
|
||||
full := inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "9c97a8a1d2c3b4a5f60718293a4b5c6d7e8f9012"}
|
||||
if !sourceIs(full, m) {
|
||||
t.Error("a full commit hash is read as a branch")
|
||||
}
|
||||
if got := followedBranch("9c97a8a"); got != "" {
|
||||
t.Errorf("a plan would re-ask the old commit %q", got)
|
||||
}
|
||||
if got := followedBranch("release"); got != "release" {
|
||||
t.Errorf("a branch is not followed as named: %q", got)
|
||||
}
|
||||
// A module that follows another branch is still not this merge's.
|
||||
if sourceIs(inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Ref: "release"}, m) {
|
||||
t.Error("a module following another branch was matched")
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,401 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A plan follows what is done to the build queue (novox/hq ADR 0219).
|
||||
//
|
||||
// Two things a person does to builds by hand would otherwise leave a plan saying something untrue:
|
||||
//
|
||||
// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder
|
||||
// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE
|
||||
// sends a reader looking for a fault that is a decision. It says what it waits on instead.
|
||||
// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a
|
||||
// failed plan's failed modules and the plan goes on from that tier as if they had built the
|
||||
// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running
|
||||
// beside it — beside it, the plan would either ask it again or stay failed on an outcome the
|
||||
// rebuild has already replaced.
|
||||
|
||||
// pauseView is whether the build seat takes work: the holders that said they are paused, and
|
||||
// whether that is every holder.
|
||||
type pauseView struct {
|
||||
Nodes []string
|
||||
All bool
|
||||
}
|
||||
|
||||
// pausedWaiting is what a plan waits on when the build seat is paused under it, or false: a plan
|
||||
// building, whose current tier has an ask outstanding, while every holder of the seat is paused.
|
||||
func pausedWaiting(p inventory.Plan, pause pauseView, now time.Time) (string, bool) {
|
||||
if p.State != inventory.PlanBuilding || !pause.All || len(pause.Nodes) == 0 || p.Tier >= len(p.Tiers) {
|
||||
return "", false
|
||||
}
|
||||
var asked *time.Time
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" && s.AskedAt != nil {
|
||||
if asked == nil || s.AskedAt.Before(*asked) {
|
||||
asked = s.AskedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
if asked == nil {
|
||||
return "", false
|
||||
}
|
||||
waited := now.Sub(*asked)
|
||||
said := fmt.Sprintf("waiting: the build seat is paused on %s (asked %s ago)",
|
||||
strings.Join(pause.Nodes, ", "), waited.Round(time.Second))
|
||||
// Not late — a pause is a decision — but a pause forgotten is a plan that never moves, so one held
|
||||
// longer than a day is named.
|
||||
if waited > pausedTooLong {
|
||||
said += " — PAUSED OVER A DAY: `resume` takes builds again"
|
||||
}
|
||||
return said, true
|
||||
}
|
||||
|
||||
// pausedTooLong is how long a plan may wait on a paused build seat before it says the pause is long.
|
||||
const pausedTooLong = 24 * time.Hour
|
||||
|
||||
// awaitsABuild is whether any open plan has an ask outstanding in its current tier — the only case
|
||||
// where the seat being paused changes what a plan says.
|
||||
func awaitsABuild(plans []inventory.Plan) bool {
|
||||
for _, p := range plans {
|
||||
if p.State != inventory.PlanBuilding || p.Tier >= len(p.Tiers) {
|
||||
continue
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "asked" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// buildSeatPause reads whether the build seat's holders take work, from what each last said on the
|
||||
// bus (link.HolderState) — read only when a plan waits on a build, so a mesh with nothing building
|
||||
// does not dial the bus to say so. Anything unreadable is said and read as not paused: a plan then
|
||||
// reads as late, which is what it said before this existed.
|
||||
func buildSeatPause(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) pauseView {
|
||||
if !awaitsABuild(plans) {
|
||||
return pauseView{}
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
seat := buildSeatAmong(entries)
|
||||
holders := holdersAmong(entries, seat)
|
||||
if len(holders) == 0 {
|
||||
return pauseView{}
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return pauseView{}
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not reach the bus to read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
defer js.Close()
|
||||
said, err := link.PausedSaid(js, seat, holders)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not read whether the build seat is paused: %v\n", err)
|
||||
return pauseView{}
|
||||
}
|
||||
return pauseOf(holders, said)
|
||||
}
|
||||
|
||||
// pauseOf is the view from what each holder said.
|
||||
func pauseOf(holders []string, said map[string]link.HolderState) pauseView {
|
||||
var v pauseView
|
||||
for _, n := range holders {
|
||||
if said[n].Paused {
|
||||
v.Nodes = append(v.Nodes, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(v.Nodes)
|
||||
v.All = len(holders) > 0 && len(v.Nodes) == len(holders)
|
||||
return v
|
||||
}
|
||||
|
||||
// failedIn is the modules of a plan's current tier that failed to build, sorted.
|
||||
func failedIn(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "failed" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan
|
||||
// that holds what this one held now (issue 254, ADR 0218).
|
||||
func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) ||
|
||||
(p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
return q, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
case p.State == inventory.PlanSuperseded:
|
||||
return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note)
|
||||
case p.Open():
|
||||
return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild <module>` asks one module again", p.ID, p.State)
|
||||
}
|
||||
if len(failedIn(p)) > 0 {
|
||||
if q, found := newerOpenPlan(p, plans); found {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s",
|
||||
p.Tier, p.ID, p.Note)
|
||||
}
|
||||
// **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or
|
||||
// sent — a newer build, and sending this one again would put the older build back on its machines.
|
||||
for _, m := range stopped {
|
||||
if q, found := newerPlanFor(m, p, plans); found {
|
||||
return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine
|
||||
// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept.
|
||||
func stoppedRollouts(p inventory.Plan) []string {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil &&
|
||||
len(s.First) > 0 && s.Why != "" {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// newerPlanFor is a plan made after this one that holds the module, superseded ones aside.
|
||||
func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) {
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) {
|
||||
continue
|
||||
}
|
||||
for _, tier := range q.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
return q, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A
|
||||
// variable so a test of a retried rollout needs no machine.
|
||||
var sendRollout = sendToEach
|
||||
|
||||
// resumed sets a failed plan building again once nothing in its tier is failed.
|
||||
func resumed(p *inventory.Plan, why string) {
|
||||
if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 {
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = why
|
||||
}
|
||||
}
|
||||
|
||||
// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at
|
||||
// that tier: what follows is the plan going on as if they had built the first time.
|
||||
func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer release()
|
||||
p, err := inv.PlanByID(ctx, id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 50)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
plans = append(plans, recent...)
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(failedIn(p)) == 0 {
|
||||
return retryRollouts(ctx, open, &p)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
failed := failedIn(p)
|
||||
var asked []string
|
||||
for _, m := range failed {
|
||||
askModule(ctx, &p, m, byName)
|
||||
if s := p.Modules[m]; s.State == "asked" {
|
||||
asked = append(asked, m+" as "+s.Build)
|
||||
}
|
||||
}
|
||||
resumed(&p, fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(failed, ", ")))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.State != inventory.PlanBuilding {
|
||||
return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note)
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does",
|
||||
p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")), nil
|
||||
}
|
||||
|
||||
// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans
|
||||
// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one.
|
||||
func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) {
|
||||
inv := open.inventory
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
defer release()
|
||||
openPlans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
recent, err := inv.RecentPlans(ctx, 20)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
p, found := planHolding(module, openPlans, recent)
|
||||
if !found {
|
||||
return false, "", nil
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
byName := map[string]inventory.Entry{}
|
||||
for _, e := range entries {
|
||||
byName[e.Manifest.Module] = e
|
||||
}
|
||||
was := p.State
|
||||
askModule(ctx, &p, module, byName)
|
||||
s := p.Modules[module]
|
||||
resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module))
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if s.State != "asked" {
|
||||
return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why)
|
||||
}
|
||||
said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome",
|
||||
s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module)
|
||||
switch {
|
||||
case was == inventory.PlanFailed && p.State == inventory.PlanBuilding:
|
||||
said += "; the plan had failed and builds again from this tier"
|
||||
case was == inventory.PlanFailed:
|
||||
said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them"
|
||||
}
|
||||
return true, said, nil
|
||||
}
|
||||
|
||||
// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not
|
||||
// yet built, else the newest failed plan whose current tier does, and that no open plan of its
|
||||
// repository supersedes.
|
||||
func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) {
|
||||
holds := func(p inventory.Plan) bool {
|
||||
if p.Tier >= len(p.Tiers) {
|
||||
return false
|
||||
}
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if m == module {
|
||||
s := p.Modules[m]
|
||||
return s == nil || s.State != "built"
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
for _, p := range openPlans {
|
||||
if holds(p) {
|
||||
return p, true
|
||||
}
|
||||
}
|
||||
sorted := append([]inventory.Plan(nil), recent...)
|
||||
sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) })
|
||||
for _, p := range sorted {
|
||||
if p.State != inventory.PlanFailed || !holds(p) {
|
||||
continue
|
||||
}
|
||||
if _, superseded := newerOpenPlan(p, openPlans); superseded {
|
||||
continue
|
||||
}
|
||||
return p, true
|
||||
}
|
||||
return inventory.Plan{}, false
|
||||
}
|
||||
|
||||
// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again,
|
||||
// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan
|
||||
// would have — the rest sent once those report they applied it, the next tier after (ADR 0218).
|
||||
func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) {
|
||||
var said []string
|
||||
for _, m := range stoppedRollouts(*p) {
|
||||
s := p.Modules[m]
|
||||
sent, err := sendRollout(ctx, open, s.First)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w",
|
||||
m, strings.Join(s.First, ", "), p.ID, err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
s.First, s.FirstAt, s.Why = sent, &now, ""
|
||||
said = append(said, m+" to "+strings.Join(sent, ", "))
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; "))
|
||||
if err := open.inventory.SavePlan(ctx, *p); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+
|
||||
"applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// `plan` tells a person what a module would open and why, from the same `why` every listens
|
||||
// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a
|
||||
// module does not need reading its manifest first.
|
||||
func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) {
|
||||
m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{
|
||||
{Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"},
|
||||
{Port: 9001, From: catalogue.FromMesh},
|
||||
}}
|
||||
got := listensLines(m)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("two listens entries, got %d: %v", len(got), got)
|
||||
}
|
||||
if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") {
|
||||
t.Errorf("the port and its why did not both appear: %q", got[0])
|
||||
}
|
||||
if strings.Contains(got[1], "—") {
|
||||
t.Errorf("a listens entry with no why should not print a dash: %q", got[1])
|
||||
}
|
||||
if !strings.Contains(got[1], "9001/tcp") {
|
||||
t.Errorf("the port still appears without a why: %q", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
|
||||
if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 {
|
||||
t.Errorf("a module with no listens should print nothing, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
|
||||
// mesh's region of a hosts file reads as the whole file.
|
||||
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
|
||||
region := shownAs(map[string]any{
|
||||
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
|
||||
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
|
||||
t.Errorf("the region is shown as %q", region)
|
||||
}
|
||||
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
|
||||
if strings.Contains(whole, "written into") {
|
||||
t.Errorf("a whole file is shown as written into: %q", whole)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: for the moment a machine hands its controller over, the container and the
|
||||
// process both run the plan timer on one store. Only the one holding the plans moves them; the other
|
||||
// leaves them alone, and moves them once they are let go.
|
||||
func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
now := time.Now().UTC()
|
||||
// Every tier done: the next step is the plan's last, and needs nothing but the store.
|
||||
plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now,
|
||||
State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The other controller: its own connections to the same store, holding the plans.
|
||||
other, err := inventory.Open(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(other.Close)
|
||||
release, err := other.HoldPlans(ctx, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(release) // before the close above: a pool waits for a connection still held
|
||||
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() {
|
||||
t.Fatalf("a controller moved a plan another held: %+v %v", p, err)
|
||||
}
|
||||
|
||||
release()
|
||||
advancePlans(ctx, open)
|
||||
if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone {
|
||||
t.Fatalf("the plan did not move once it was let go: %+v %v", p, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// One machine that cannot be worked out is not a reason to leave the mesh unconverged.
|
||||
//
|
||||
// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on
|
||||
// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine
|
||||
// from being sent anything — machines with no relation to the fault, and nothing wrong with them.
|
||||
// The failure and the punishment were on different machines.
|
||||
//
|
||||
// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
}
|
||||
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||
})
|
||||
|
||||
var told []string
|
||||
for _, s := range sending {
|
||||
told = append(told, s.node)
|
||||
}
|
||||
if strings.Join(told, ",") != "home-server,laptop" {
|
||||
t.Errorf("a machine with nothing wrong with it was not sent: %v", told)
|
||||
}
|
||||
if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") ||
|
||||
!strings.Contains(refusals[0], "acme-ca") {
|
||||
t.Errorf("the machine that could not be worked out was not named with its reason: %v",
|
||||
refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A push that skipped a machine still ends badly, and says what was sent.
|
||||
//
|
||||
// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command
|
||||
// exits non-zero — but it says how many machines it did reach, because the old message ("nothing
|
||||
// was sent") was the very claim that had become untrue.
|
||||
func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
if err := couldNotBeResolved(nil, 3); err != nil {
|
||||
t.Fatalf("a push that resolved every machine reported a problem: %v", err)
|
||||
}
|
||||
|
||||
err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2)
|
||||
if err == nil {
|
||||
t.Fatal("a push that could not work out a machine reported success")
|
||||
}
|
||||
said := err.Error()
|
||||
if strings.Contains(said, "nothing was sent") {
|
||||
t.Errorf("the push says nothing was sent, and it sent two machines: %q", said)
|
||||
}
|
||||
for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the refusal does not say %q: %q", want, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (int64, error) {
|
||||
var n int64
|
||||
return func(string) (int64, error) { n++; return n, nil }
|
||||
}
|
||||
@@ -0,0 +1,709 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219).
|
||||
//
|
||||
// Seen whole — what waits, what runs where and for how long, what was handed out as often as it
|
||||
// may be and never settled — and changed through the controller: an ask cancelled, the queue
|
||||
// cleared, a module rebuilt, a build replayed. What one machine is running is that machine's
|
||||
// holder's to end or pause, and the controller asks it (`kill`, `pause`, `resume`).
|
||||
//
|
||||
// **Everything that drops an ask leaves a failed outcome for it**, taken in exactly as a build that
|
||||
// failed is (takeIn, then the plan): a plan waiting on an ask a person removed fails, saying so,
|
||||
// rather than waiting for ever on an answer nobody will give.
|
||||
|
||||
// holderAsks is how long a holder's verb is waited for; killAnswer how long its kill is — longer
|
||||
// than the holder's worst case (its two passes removing containers and its wait between, 50s).
|
||||
const (
|
||||
holderAsks = 15 * time.Second
|
||||
killAnswer = 75 * time.Second
|
||||
)
|
||||
|
||||
// dialTheBus opens the controller's own connection, for a command that reads or changes the queue.
|
||||
func dialTheBus() (*broker.JetStream, error) {
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot reach the bus: %w", err)
|
||||
}
|
||||
return js, nil
|
||||
}
|
||||
|
||||
// queueCommand prints every ask in the build seat's work queue.
|
||||
func queueCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("queue", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the queue as JSON")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(q, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
fmt.Print(queueText(q, time.Now()))
|
||||
return nil
|
||||
}
|
||||
|
||||
// queueText is the queue as a person reads it: a summary line, then each kind in queue order.
|
||||
// Never what an ask carries as `held` — that is every artifact the mesh has built.
|
||||
func queueText(q link.Queue, now time.Time) string {
|
||||
var b strings.Builder
|
||||
waiting, running, dead := q.Of(link.AskWaiting), q.Of(link.AskInFlight), q.Of(link.AskDead)
|
||||
fmt.Fprintf(&b, "%s: %d waiting, %d in flight, %d dead\n", q.Seat, len(waiting), len(running), len(dead))
|
||||
ago := func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "asked at an unknown time"
|
||||
}
|
||||
return "asked " + now.Sub(t).Round(time.Second).String() + " ago"
|
||||
}
|
||||
what := func(a link.QueuedAsk) string {
|
||||
s := a.Repository
|
||||
if a.Path != "" {
|
||||
s += " at " + a.Path
|
||||
}
|
||||
if a.Ref != "" {
|
||||
s += " on " + a.Ref
|
||||
}
|
||||
return s
|
||||
}
|
||||
if len(running) > 0 {
|
||||
fmt.Fprintln(&b, "\nin flight:")
|
||||
for _, a := range running {
|
||||
where := "taken, not yet said where"
|
||||
switch {
|
||||
case a.On != "":
|
||||
where = fmt.Sprintf("on %s for %s", a.On, now.Sub(a.Started).Round(time.Second))
|
||||
case a.Was != "":
|
||||
where = fmt.Sprintf("handed back by %s, to be handed out again", a.Was)
|
||||
}
|
||||
fmt.Fprintf(&b, " %-26s %s — %s, %s (seq %d)\n", a.ID, what(a), where, ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
fmt.Fprintln(&b, "\nwaiting:")
|
||||
for _, a := range waiting {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
if len(dead) > 0 {
|
||||
fmt.Fprintf(&b, "\ndead — handed out as often as the worker allows (%d) and never settled, still held:\n", q.MaxDeliver)
|
||||
for _, a := range dead {
|
||||
fmt.Fprintf(&b, " %-26s %s — %s (seq %d)\n", a.ID, what(a), ago(a.AskedAt), a.Seq)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(q.Asks) == 0:
|
||||
fmt.Fprintln(&b, "nothing is asked of it")
|
||||
default:
|
||||
fmt.Fprintln(&b, "\n`cancel <id>` drops a waiting or dead ask, `clear` every waiting one, `kill <id>` ends one in flight")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// cancelCommand drops one waiting or dead ask.
|
||||
func cancelCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("cancel <build id>")
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ask, found := q.Find(args[0])
|
||||
if !found {
|
||||
return fmt.Errorf("%s is not in the %s queue: it was built, cancelled, or never asked — `builds` says "+
|
||||
"what came of it", args[0], seat)
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancelAsk drops one ask from the queue and records it failed, cancelled by hand.
|
||||
//
|
||||
// **In this order, and each step for a reason** (novox/hq ADR 0219):
|
||||
// 1. an ask a machine says it is building is refused: deleting its message ends nothing a machine
|
||||
// is running — that is `kill`, on the machine running it;
|
||||
// 2. its id goes into the seat's cancelled set, so a holder that fetches it from now on ends it;
|
||||
// 3. for a waiting ask, the worker is read again: one handed out since the queue was read was
|
||||
// taken in the moment of cancelling, and the cancel is withdrawn and refused — the holder either
|
||||
// read the mark first and ends it as cancelled, or is building it;
|
||||
// 4. for an ask the worker counts handed out that no machine is building — taken and not yet said,
|
||||
// handed back after a restart, or past its deliveries while nobody pulls — the holder's own look
|
||||
// at the set is waited out, and a start heard since withdraws and refuses the cancel: a holder
|
||||
// that took it before the mark is building it, and only `kill` ends that;
|
||||
// 5. the message is deleted by its sequence;
|
||||
// 6. the failed outcome is taken in as any failed build's is, and the plan that asked fails.
|
||||
func cancelAsk(ctx context.Context, js *broker.JetStream, open *stores, seat string, ask link.QueuedAsk) (string, error) {
|
||||
if ask.State == link.AskInFlight && ask.On != "" {
|
||||
return "", fmt.Errorf("%s is in flight on %s: cancelling drops an ask nobody is building. `kill %s` "+
|
||||
"ends the build where it runs", ask.ID, ask.On, ask.ID)
|
||||
}
|
||||
if err := link.MarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
withdraw := func() {
|
||||
if err := link.UnmarkCancelled(ctx, js, seat, ask.ID); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "could not withdraw the cancel of %s: %v — a holder taking it ends it as cancelled\n", ask.ID, err)
|
||||
}
|
||||
}
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: seat, Accepts: []string{"build"}})
|
||||
switch ask.State {
|
||||
case link.AskWaiting:
|
||||
if taken, err := takenSince(js, worker, ask.Seq); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if taken {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s was taken by a holder as it was cancelled, so the cancel is withdrawn. If the "+
|
||||
"holder read it first it ends the ask as %s and its outcome says so; otherwise it is building — "+
|
||||
"`queue` says which, and `kill %s` ends it", ask.ID, link.CancelledByHand, ask.ID)
|
||||
}
|
||||
case link.AskInFlight:
|
||||
if started, err := startedSince(ctx, js, seat, ask); err != nil {
|
||||
withdraw()
|
||||
return "", err
|
||||
} else if started != "" {
|
||||
withdraw()
|
||||
return "", fmt.Errorf("%s has started on %s since it was read, so the cancel is withdrawn: `kill %s` "+
|
||||
"ends it there", ask.ID, started, ask.ID)
|
||||
}
|
||||
}
|
||||
if err := js.Context().DeleteMsg(worker.Stream, ask.Seq); err != nil && !errors.Is(err, nats.ErrMsgNotFound) &&
|
||||
!errors.Is(err, jetstream.ErrMsgNotFound) && !strings.Contains(err.Error(), "no message found") {
|
||||
return "", fmt.Errorf("%s is marked cancelled and could not be deleted from the queue (seq %d): %w — a "+
|
||||
"holder taking it ends it as cancelled", ask.ID, ask.Seq, err)
|
||||
}
|
||||
recordCancelled(ctx, open, ask)
|
||||
return fmt.Sprintf("cancelled %s (%s, %s): deleted from the %s queue and recorded failed, %s — a plan "+
|
||||
"that asked for it fails with that", ask.ID, ask.Repository, ask.State, seat, link.CancelledByHand), nil
|
||||
}
|
||||
|
||||
// holderLooks is how long a cancel waits for a holder that took the ask before the mark to say it
|
||||
// started: longer than a holder's look at the cancelled set (3s) and its start that follows.
|
||||
var holderLooks = 5 * time.Second
|
||||
|
||||
// startedSince waits out a holder's look at the cancelled set and says the machine that started the
|
||||
// ask since it was read, or nothing. A start it ended as cancelled comes with its outcome and is not
|
||||
// a start of a build.
|
||||
func startedSince(ctx context.Context, js *broker.JetStream, seat string, ask link.QueuedAsk) (string, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", ctx.Err()
|
||||
case <-time.After(holderLooks):
|
||||
}
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if !ask.AskedAt.IsZero() {
|
||||
since = ask.AskedAt.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s, ok := heard.Started[ask.ID]
|
||||
if !ok || heard.Outcomes[ask.ID] {
|
||||
return "", nil
|
||||
}
|
||||
at, _ := time.Parse(time.RFC3339Nano, s.At)
|
||||
if ask.Started.IsZero() || at.After(ask.Started) {
|
||||
return s.On, nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// takenSince is whether the worker has handed out the ask at this sequence.
|
||||
func takenSince(js *broker.JetStream, worker broker.Consumer, seq uint64) (bool, error) {
|
||||
info, err := js.Context().ConsumerInfo(worker.Stream, worker.Name)
|
||||
if errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("cannot read the worker of the queue again: %w", err)
|
||||
}
|
||||
return info.Delivered.Stream >= seq, nil
|
||||
}
|
||||
|
||||
// recordCancelled takes the failed outcome in the way the daemon takes in any build's: recorded,
|
||||
// then the plan that asked for it — by the id it asked with, or by repository and path.
|
||||
func recordCancelled(ctx context.Context, open *stores, ask link.QueuedAsk) {
|
||||
r := ask.Request
|
||||
result := link.BuildResult{ID: ask.ID, Repository: ask.Repository, Path: ask.Path, Ref: ask.Ref,
|
||||
Source: r.Source, DryRun: r.DryRun, Failed: link.CancelledByHand}
|
||||
_ = builds{open.inventory, open}.Built(ctx, result)
|
||||
}
|
||||
|
||||
// clearCommand cancels every waiting ask, and with --dead every dead one too.
|
||||
func clearCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("clear", flag.ContinueOnError)
|
||||
dead := set.Bool("dead", false, "the dead asks too")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
said, err := clearQueue(ctx, js, open, seat, *dead)
|
||||
fmt.Print(said)
|
||||
return err
|
||||
}
|
||||
|
||||
// clearQueue cancels what clear names, each as `cancel` would, and never anything in flight.
|
||||
func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat string, dead bool) (string, error) {
|
||||
q, err := link.ReadQueue(ctx, js, seat)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var b strings.Builder
|
||||
cancelled, refused := 0, 0
|
||||
for _, a := range q.Asks {
|
||||
if a.State == link.AskInFlight || (a.State == link.AskDead && !dead) {
|
||||
continue
|
||||
}
|
||||
said, err := cancelAsk(ctx, js, open, seat, a)
|
||||
if err != nil {
|
||||
refused++
|
||||
fmt.Fprintf(&b, " %s: %v\n", a.ID, err)
|
||||
continue
|
||||
}
|
||||
cancelled++
|
||||
fmt.Fprintf(&b, " %s\n", said)
|
||||
}
|
||||
what := "waiting"
|
||||
if dead {
|
||||
what = "waiting and dead"
|
||||
}
|
||||
fmt.Fprintf(&b, "%d %s ask(s) cancelled", cancelled, what)
|
||||
if refused > 0 {
|
||||
fmt.Fprintf(&b, ", %d could not be", refused)
|
||||
}
|
||||
fmt.Fprintln(&b)
|
||||
if n := len(q.Of(link.AskInFlight)); n > 0 {
|
||||
fmt.Fprintf(&b, "%d in flight, left running: `kill <id>` ends one where it runs\n", n)
|
||||
}
|
||||
if n := len(q.Of(link.AskDead)); n > 0 && !dead {
|
||||
fmt.Fprintf(&b, "%d dead, left: `clear --dead` cancels them too\n", n)
|
||||
}
|
||||
if refused > 0 {
|
||||
return b.String(), fmt.Errorf("%d ask(s) could not be cancelled", refused)
|
||||
}
|
||||
return b.String(), nil
|
||||
}
|
||||
|
||||
// rebuildCommand asks the module's current source again — or, given a build's id, that build's
|
||||
// repository, path and ref — under a new id.
|
||||
func rebuildCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("rebuild <module | build id>")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var source buildSource
|
||||
var path, ref, module string
|
||||
if b, found, err := inv.BuildByID(ctx, args[0]); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
source, module = sourceOfBuild(b, entries)
|
||||
path, ref = b.Path, b.Ref
|
||||
// **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue
|
||||
// 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll
|
||||
// that commit out over everything since. Building that commit again is `replay`, which says
|
||||
// what it would do and refuses to register it while anything newer is asked or registered.
|
||||
if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" {
|
||||
ref = followedBranch(e.Source.Ref)
|
||||
follows := ref
|
||||
if follows == "" {
|
||||
follows = "the repository's default branch"
|
||||
}
|
||||
fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+
|
||||
"builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID)
|
||||
}
|
||||
} else if e, known := entryNamed(entries, args[0]); known {
|
||||
// As a plan asks it: the branch it follows, never a commit a build once named (issue 215).
|
||||
source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}
|
||||
path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module
|
||||
} else {
|
||||
return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0])
|
||||
}
|
||||
|
||||
// **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the
|
||||
// plan would ask it again, or stay failed on an outcome a rebuild has replaced.
|
||||
if module != "" {
|
||||
joined, said, err := joinAPlan(ctx, open, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if joined {
|
||||
fmt.Println(said)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
id, err := askABuild(ctx, source, path, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("rebuild asked as %s\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// entryNamed is the catalogue's entry for a module.
|
||||
func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) {
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == name {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
|
||||
// sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own
|
||||
// source when the build is of one — on its seat, so the outcome registers it as the mesh records it
|
||||
// (ADR 0111) — else the repository as it was cloned.
|
||||
func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, string) {
|
||||
for _, e := range entries {
|
||||
if (b.Module != "" && e.Manifest.Module == b.Module) ||
|
||||
(b.Module == "" && repositoryMatches(e.Source.Repository, b.Repository) && e.Source.Path == b.Path) {
|
||||
return buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat}, e.Manifest.Module
|
||||
}
|
||||
}
|
||||
return buildSource{Repository: b.Repository}, b.Module
|
||||
}
|
||||
|
||||
// replayCommand asks a recorded build's repository and path again at the commit it built.
|
||||
func replayCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("replay", flag.ContinueOnError)
|
||||
register := set.Bool("register", false, "register what it builds, as any build is")
|
||||
older := set.Bool("older", false, "with --register: even though a newer build of the module is registered")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("replay <build id> [--register [--older]]")
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
b, found, err := inv.BuildByID(ctx, positionals[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("no build %s is recorded", positionals[0])
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, module := sourceOfBuild(b, entries)
|
||||
var history []inventory.Build
|
||||
if module != "" {
|
||||
if history, err = inv.Builds(ctx, module, 100); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// What is asked of the module and not yet answered, when the replay would be registered.
|
||||
var outstanding []string
|
||||
if *register {
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx))
|
||||
js.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outstanding = outstandingFor(module, b.Repository, b.Path, q, plans)
|
||||
}
|
||||
if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil {
|
||||
return err
|
||||
}
|
||||
id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(replaySaid(b, module, id, *register))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207).
|
||||
//
|
||||
// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it
|
||||
// was asked, which is what orders builds of one module (issue 219). So a registered replay of an
|
||||
// older commit is newer than everything since, and would roll that older commit out as the module's
|
||||
// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless
|
||||
// --register says otherwise, and --register is refused when a newer build of a different commit is
|
||||
// registered, unless --older says that is the point.
|
||||
//
|
||||
// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the
|
||||
// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks,
|
||||
// and their builds — made from newer source — would be recorded and never registered (issue 219
|
||||
// orders by ask), the plan waiting on them sending the older commit instead.
|
||||
func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool,
|
||||
outstanding []string) error {
|
||||
if b.Commit == "" {
|
||||
return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+
|
||||
"nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID)
|
||||
}
|
||||
if older && !register {
|
||||
return errors.New("--older only says what --register may do; a dry run registers nothing")
|
||||
}
|
||||
if register && len(outstanding) > 0 {
|
||||
return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+
|
||||
"replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look",
|
||||
orNone(module), strings.Join(outstanding, "; "), b.ID)
|
||||
}
|
||||
if !register || older {
|
||||
return nil
|
||||
}
|
||||
for _, h := range history {
|
||||
if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit {
|
||||
continue
|
||||
}
|
||||
if h.AskedOrAt().After(b.AskedOrAt()) {
|
||||
return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+
|
||||
"would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+
|
||||
"without --register looks at it; --register --older registers it anyway",
|
||||
module, h.ID, short(h.Commit), short(b.Commit), module, b.ID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replaySaid is what a replay prints once asked: what it builds, and what becomes of the outcome.
|
||||
func replaySaid(b inventory.Build, module, id string, register bool) string {
|
||||
what := b.Repository
|
||||
if module != "" {
|
||||
what = module
|
||||
}
|
||||
said := fmt.Sprintf("replaying %s (%s) at %s as %s", b.ID, what, short(b.Commit), id)
|
||||
if !register {
|
||||
return said + "\n a dry run: the outcome is looked at and not taken in — nothing is recorded or " +
|
||||
"registered, and nothing is sent. `builds --log " + id + "` follows it; `--register` registers it"
|
||||
}
|
||||
return said + "\n registered when it is built, as the module's current version — newer than every build " +
|
||||
"asked before now — and rolled out as its policy says. `builds --log " + id + "` follows it"
|
||||
}
|
||||
|
||||
// killCommand finds the machine running a build and asks its holder to end it.
|
||||
func killCommand(ctx context.Context, args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("kill <build id>")
|
||||
}
|
||||
id := args[0]
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
since := time.Now().Add(-7 * 24 * time.Hour)
|
||||
if at, ok := link.BuildAskedAt(id); ok {
|
||||
since = at.Add(-time.Minute)
|
||||
}
|
||||
heard, err := link.ReadBuildEvents(ctx, js, seat, since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
started, ok := heard.Started[id]
|
||||
if !ok {
|
||||
return fmt.Errorf("no machine has said it started %s: if it waits in the queue, `cancel %s` drops it", id, id)
|
||||
}
|
||||
if heard.Outcomes[id] {
|
||||
return fmt.Errorf("%s has already ended on %s — `builds` says how", id, started.On)
|
||||
}
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, "kill", started.On, map[string]string{"id": id}, killAnswer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return printHolderAnswer(started.On, answer)
|
||||
}
|
||||
|
||||
// pauseCommand asks one machine's holder, or every holder's, to pause or resume.
|
||||
func pauseCommand(ctx context.Context, verb string, args []string) error {
|
||||
if len(args) > 1 {
|
||||
return fmt.Errorf("%s [node]", verb)
|
||||
}
|
||||
js, err := dialTheBus()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer js.Close()
|
||||
seat := buildSeatHeld(ctx)
|
||||
nodes := args
|
||||
if len(nodes) == 0 {
|
||||
if nodes, err = buildSeatHolders(ctx, seat); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return fmt.Errorf("nothing holds %s, so there is nothing to %s", seat, verb)
|
||||
}
|
||||
}
|
||||
failed := 0
|
||||
for _, node := range nodes {
|
||||
answer, err := link.AskSeatTool(ctx, js.Conn(), seat, verb, node, map[string]string{}, holderAsks)
|
||||
if err != nil {
|
||||
fmt.Printf("%s: %v\n", node, err)
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
if err := printHolderAnswer(node, answer); err != nil {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("%d of %d machine(s) did not %s", failed, len(nodes), verb)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printHolderAnswer prints what a holder said, or its refusal as the command's failure.
|
||||
func printHolderAnswer(node string, answer link.Answer) error {
|
||||
if answer.Error != "" {
|
||||
fmt.Printf("%s: %s\n", node, answer.Error)
|
||||
return errors.New(answer.Error)
|
||||
}
|
||||
var said struct {
|
||||
Said string `json:"said"`
|
||||
}
|
||||
if json.Unmarshal(answer.Result, &said) == nil && said.Said != "" {
|
||||
fmt.Printf("%s: %s\n", node, said.Said)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s: %s\n", node, string(answer.Result))
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildSeatHolders is every machine an assigned module holding the build seat runs on.
|
||||
func buildSeatHolders(ctx context.Context, seat string) ([]string, error) {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer open.Close()
|
||||
entries, err := open.inventory.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return holdersAmong(entries, seat), nil
|
||||
}
|
||||
|
||||
// holdersAmong is the machines of every catalogued module claiming the seat, sorted, once each.
|
||||
func holdersAmong(entries []inventory.Entry, seat string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.Manifest.ClaimsSeat(seat) {
|
||||
continue
|
||||
}
|
||||
for _, n := range e.On {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue,
|
||||
// by repository and path, and every open plan holding it not yet built.
|
||||
func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string {
|
||||
var out []string
|
||||
for _, a := range q.Asks {
|
||||
if repositoryMatches(a.Repository, repository) && a.Path == path {
|
||||
out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State))
|
||||
}
|
||||
}
|
||||
if module == "" {
|
||||
return out
|
||||
}
|
||||
for _, p := range plans {
|
||||
if !p.Open() {
|
||||
continue
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
if m == module {
|
||||
if st := p.Modules[m]; st == nil || st.State != "built" {
|
||||
out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,772 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it.
|
||||
//
|
||||
// docker run -d --rm --name bq-nats -p 14294:4222 nats:2.10-alpine -js
|
||||
// make postgres PG_PORT=55566 PG_CONTAINER=bq-pg
|
||||
// MESH_TEST_NATS=nats://127.0.0.1:14294 \
|
||||
// MESH_TEST_POSTGRES='postgres://postgres:check@127.0.0.1:55566/postgres?sslmode=disable' \
|
||||
// go test ./cmd/mesh-controller/ -run 'Queue|Cancel|Clear|Retry|Rebuild|Replay|Paused'
|
||||
|
||||
// asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked.
|
||||
func asksRecorded(t *testing.T) *[]string {
|
||||
t.Helper()
|
||||
var asked []string
|
||||
was := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asked)) * time.Millisecond))
|
||||
asked = append(asked, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = was })
|
||||
return &asked
|
||||
}
|
||||
|
||||
// twoTiers registers two modules, b standing on a, each with a source a plan asks.
|
||||
func twoTiers(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"a", "b"} {
|
||||
if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"},
|
||||
inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan is retried: its failed module asked again under a new id, the plan building at that
|
||||
// tier, and when that build comes in the plan goes on and asks its next tier.
|
||||
func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1",
|
||||
Why: link.KilledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s and a is %+v", p.State, a)
|
||||
}
|
||||
if !strings.Contains(said, a.Build) {
|
||||
t.Errorf("retry does not say the new id: %q", said)
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
|
||||
// The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows.
|
||||
planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1")
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding {
|
||||
t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note)
|
||||
}
|
||||
asking, _ := link.BuildAskedAt(a.Build)
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build)
|
||||
p, err = open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" {
|
||||
t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// What retry refuses, and says why.
|
||||
func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
plan := func(id, state string, created time.Time) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: id + "c0ffee",
|
||||
Created: created, State: state, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
}
|
||||
failed := plan("plan-1", inventory.PlanFailed, at)
|
||||
for _, c := range []struct {
|
||||
p inventory.Plan
|
||||
others []inventory.Plan
|
||||
says string
|
||||
}{
|
||||
{plan("plan-d", inventory.PlanDone, at), nil, "is done"},
|
||||
{plan("plan-s", inventory.PlanSuperseded, at), nil, "was"},
|
||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||
} {
|
||||
err := retryRefusal(c.p, c.others)
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||
}
|
||||
}
|
||||
stopped := failed
|
||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
stopped.Note = "a stopped at its first machine"
|
||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// Another branch's newer plan, an older one, and a failed one do not supersede it.
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
other.Branch = "release"
|
||||
if err := retryRefusal(failed, []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour)),
|
||||
plan("plan-4", inventory.PlanFailed, at.Add(time.Hour))}); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone.
|
||||
func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
before := time.Now().UTC().Add(-time.Hour)
|
||||
failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a failed to build in tier 0",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}}
|
||||
if err := open.inventory.SavePlan(ctx, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"a"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" {
|
||||
t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
// b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone.
|
||||
if err := rebuildCommand(ctx, []string{"b"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil {
|
||||
t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 2 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module.
|
||||
func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed,
|
||||
Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}}
|
||||
newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found {
|
||||
t.Fatal("joined a failed plan a newer open one supersedes")
|
||||
}
|
||||
if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" {
|
||||
t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID)
|
||||
}
|
||||
built := failed
|
||||
built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
if _, found := planHolding("a", nil, []inventory.Plan{built}); found {
|
||||
t.Fatal("joined a plan that has the module built")
|
||||
}
|
||||
}
|
||||
|
||||
// replay is a dry run unless registered, and registering an older commit than one registered since
|
||||
// is refused unless --older says it is meant (novox/hq issue 207).
|
||||
func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked}
|
||||
newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)}
|
||||
history := []inventory.Build{newer, old}
|
||||
|
||||
if err := replayRefusal(old, "a", history, false, false, nil); err != nil {
|
||||
t.Errorf("a dry run was refused: %v", err)
|
||||
}
|
||||
err := replayRefusal(old, "a", history, true, false, nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") {
|
||||
t.Errorf("registering an older commit than the one registered was not refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, nil); err != nil {
|
||||
t.Errorf("--register --older was refused: %v", err)
|
||||
}
|
||||
if err := replayRefusal(newer, "a", history, true, false, nil); err != nil {
|
||||
t.Errorf("registering the newest build again was refused: %v", err)
|
||||
}
|
||||
// A newer build of the same commit, or one that failed, is not a newer version to roll back from.
|
||||
same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)}
|
||||
broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)}
|
||||
if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil {
|
||||
t.Errorf("refused for a newer build of the same commit or a failed one: %v", err)
|
||||
}
|
||||
if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil {
|
||||
t.Error("a build that recorded no commit was replayed")
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, true, nil); err == nil {
|
||||
t.Error("--older without --register was taken")
|
||||
}
|
||||
// **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan
|
||||
// holding it unbuilt, would be replaced by a replay asked now (issue 219).
|
||||
q := link.Queue{Asks: []link.QueuedAsk{
|
||||
{ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting},
|
||||
{ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting},
|
||||
{ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting},
|
||||
}}
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}},
|
||||
{ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
}
|
||||
outstanding := outstandingFor("a", "novox/a", "", q, plans)
|
||||
if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") {
|
||||
t.Fatalf("outstanding: %v", outstanding)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") {
|
||||
t.Errorf("registered over an outstanding ask: %v", err)
|
||||
}
|
||||
if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil {
|
||||
t.Errorf("a dry run was refused for what is outstanding: %v", err)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") {
|
||||
t.Errorf("a dry replay does not say what it is: %q", said)
|
||||
}
|
||||
if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") {
|
||||
t.Errorf("a registered replay does not say what it does: %q", said)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat
|
||||
// paused on some holders only is not a reason the plan is waiting.
|
||||
func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
asked := now.Add(-12 * time.Minute)
|
||||
p := inventory.Plan{ID: "plan-p", Repository: "novox/a", Commit: "c0ffee", State: inventory.PlanBuilding,
|
||||
Updated: now.Add(-2 * time.Hour), Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}}
|
||||
|
||||
all := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}, "g14": {Paused: true}})
|
||||
line := planLineWith(p, now, all)
|
||||
if !strings.Contains(line, "waiting: the build seat is paused on ace, g14 (asked 12m0s ago)") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan on a paused seat reads %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, all)[0]; st.Late || !strings.Contains(st.Waiting, "paused on ace, g14") {
|
||||
t.Errorf("status --json says %+v", st)
|
||||
}
|
||||
if _, late := openPlans([]inventory.Plan{p}, all); late != 0 {
|
||||
t.Errorf("a plan waiting on a paused seat counted late")
|
||||
}
|
||||
|
||||
longAgo := now.Add(-25 * time.Hour)
|
||||
forgotten := p
|
||||
forgotten.Modules = map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &longAgo}}
|
||||
if line := planLineWith(forgotten, now, all); !strings.Contains(line, "PAUSED OVER A DAY") || strings.Contains(line, "LATE") {
|
||||
t.Errorf("a plan paused over a day reads %q", line)
|
||||
}
|
||||
|
||||
some := pauseOf([]string{"g14", "ace"}, map[string]link.HolderState{"ace": {Paused: true}})
|
||||
if some.All || !reflect.DeepEqual(some.Nodes, []string{"ace"}) {
|
||||
t.Fatalf("%+v", some)
|
||||
}
|
||||
if line := planLineWith(p, now, some); !strings.Contains(line, "LATE") {
|
||||
t.Errorf("a seat paused on one holder of two made the plan not late: %q", line)
|
||||
}
|
||||
if st := planStatuses([]inventory.Plan{p}, now, some)[0]; !st.Late {
|
||||
t.Errorf("status --json: %+v", st)
|
||||
}
|
||||
// A plan rolling out, or with nothing asked, is not waiting on the seat.
|
||||
rolling := p
|
||||
rolling.State = inventory.PlanRolling
|
||||
if _, paused := pausedWaiting(rolling, all, now); paused {
|
||||
t.Error("a rolling plan reads as waiting on the build seat")
|
||||
}
|
||||
}
|
||||
|
||||
// The queue's verbs are the controller seat's, each to the command it names.
|
||||
func TestTheQueueVerbsRunTheirCommands(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
verb string
|
||||
args map[string]any
|
||||
want []string
|
||||
}{
|
||||
{"queue", nil, []string{"queue"}},
|
||||
{"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}},
|
||||
{"clear", nil, []string{"clear"}},
|
||||
{"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}},
|
||||
{"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}},
|
||||
{"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}},
|
||||
{"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}},
|
||||
{"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}},
|
||||
{"pause", nil, []string{"pause"}},
|
||||
{"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}},
|
||||
{"plans", map[string]any{"retry": "plan-1"}, []string{"plans", "retry", "plan-1"}},
|
||||
} {
|
||||
got, err := argvFor(c.verb, c.args)
|
||||
if err != nil || !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("cancel", nil); err == nil {
|
||||
t.Error("cancel without an id was taken")
|
||||
}
|
||||
declared := map[string]bool{}
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
declared[v.Name] = true
|
||||
}
|
||||
for _, v := range []string{"queue", "cancel", "clear", "rebuild", "replay", "kill", "pause", "resume"} {
|
||||
if !declared[v] {
|
||||
t.Errorf("%s is not a verb of the controller seat", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- against a real bus -----------------------------------------------------------------------
|
||||
|
||||
// aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller
|
||||
// pointed at it, and a function that asks the seat one build.
|
||||
func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) {
|
||||
t.Helper()
|
||||
url := os.Getenv("MESH_TEST_NATS")
|
||||
if url == "" {
|
||||
t.Skip("MESH_TEST_NATS unset")
|
||||
}
|
||||
t.Setenv(broker.NATSVar, url)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"},
|
||||
Emits: []string{"started", "built", "log.*", "paused.*"}}
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{
|
||||
link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT")
|
||||
_ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine))
|
||||
_ = js.Context().PurgeStream(broker.EventsStream)
|
||||
})
|
||||
ask := func(id, repository string) link.BuildRequest {
|
||||
r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}}
|
||||
body, _ := json.Marshal(r)
|
||||
if _, err := js.Context().Publish(link.BuildWork(), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
return js, ask
|
||||
}
|
||||
|
||||
// deadOne takes the oldest ask from the worker and hands it back as often as the worker allows.
|
||||
func deadOne(t *testing.T, js *broker.JetStream) {
|
||||
t.Helper()
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
for i := 0; i < worker.MaxDeliver; i++ {
|
||||
msgs, err := sub.Fetch(1, nats.MaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatalf("delivery %d: %v", i+1, err)
|
||||
}
|
||||
_ = msgs[0].Nak()
|
||||
}
|
||||
// One more pull, as a holder always has one waiting: the server finds the ask past its deliveries
|
||||
// then, and stops counting it pending.
|
||||
if msgs, _ := sub.Fetch(1, nats.MaxWait(time.Second)); len(msgs) > 0 {
|
||||
t.Fatalf("an ask past its deliveries was delivered again")
|
||||
}
|
||||
}
|
||||
|
||||
// cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan
|
||||
// that asked for it, matched by the id, fails with it; an ask the plan's records name no module for
|
||||
// is still found.
|
||||
func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
twoTiers(t, open)
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
asked, _ := link.BuildAskedAt(id)
|
||||
plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") ||
|
||||
strings.Contains(text, "secret-ish") {
|
||||
t.Errorf("the queue says:\n%s", text)
|
||||
}
|
||||
|
||||
if err := cancelCommand(ctx, []string{id}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("the ask is still queued: %+v", q.Asks)
|
||||
}
|
||||
if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled {
|
||||
t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err)
|
||||
}
|
||||
b, found, err := open.inventory.BuildByID(ctx, id)
|
||||
if err != nil || !found || b.Failed != link.CancelledByHand {
|
||||
t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand {
|
||||
t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"])
|
||||
}
|
||||
if err := cancelCommand(ctx, []string{id}); err == nil {
|
||||
t.Error("an ask cancelled already was cancelled again")
|
||||
}
|
||||
}
|
||||
|
||||
// clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight.
|
||||
func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
start := time.Now()
|
||||
dead := link.NewBuildID(start)
|
||||
ask(dead, "https://forge.example/novox/dead.git")
|
||||
deadOne(t, js)
|
||||
var waiting []string
|
||||
for i := 1; i <= 2; i++ {
|
||||
id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond))
|
||||
waiting = append(waiting, id)
|
||||
ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i))
|
||||
}
|
||||
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 {
|
||||
t.Fatalf("the queue reads %+v", q)
|
||||
}
|
||||
said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") {
|
||||
t.Errorf("clear said:\n%s", said)
|
||||
}
|
||||
q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead {
|
||||
t.Fatalf("after clear the queue is %+v", q.Asks)
|
||||
}
|
||||
if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 {
|
||||
t.Fatalf("clear --dead left %+v", q.Asks)
|
||||
}
|
||||
for _, id := range append(waiting, dead) {
|
||||
if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand {
|
||||
t.Errorf("%s is recorded as %+v", id, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// An ask in flight is not cancelled: kill ends it where it runs.
|
||||
func TestCancelRefusesAnAskInFlight(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "ace" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
_, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a)
|
||||
if err == nil || !strings.Contains(err.Error(), "kill "+id) {
|
||||
t.Fatalf("an ask in flight was cancelled: %v", err)
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a refused cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
|
||||
// kill finds the machine from the build's start and asks that machine's holder; pause asks the
|
||||
// machine named. Each prints what the holder answered, and a refusal is the command's failure.
|
||||
func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
js, _ := aBuildQueue(t)
|
||||
ctx := t.Context()
|
||||
asked := map[string]string{}
|
||||
handlers := map[string]link.ToolHandler{
|
||||
"kill": func(_ context.Context, raw json.RawMessage) (any, error) {
|
||||
var args struct{ ID string }
|
||||
_ = json.Unmarshal(raw, &args)
|
||||
asked["kill"] = args.ID
|
||||
if args.ID != "build-running" {
|
||||
return nil, fmt.Errorf("ace is not building %s", args.ID)
|
||||
}
|
||||
return map[string]any{"said": "killed " + args.ID}, nil
|
||||
},
|
||||
"pause": func(context.Context, json.RawMessage) (any, error) {
|
||||
asked["pause"] = "ace"
|
||||
return map[string]any{"said": "ace is paused"}, nil
|
||||
},
|
||||
}
|
||||
stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer stop()
|
||||
for _, id := range []string{"build-running", "build-other"} {
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-running"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked["kill"] != "build-running" {
|
||||
t.Fatalf("the holder was asked %v", asked)
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-other"}); err == nil {
|
||||
t.Error("the holder's refusal was not the command's")
|
||||
}
|
||||
if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") {
|
||||
t.Errorf("a build nobody started: %v", err)
|
||||
}
|
||||
if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" {
|
||||
t.Fatalf("pause: %v %v", err, asked)
|
||||
}
|
||||
if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil {
|
||||
t.Error("a machine nothing answers on was resumed")
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var sentTo [][]string
|
||||
was := sendRollout
|
||||
sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) {
|
||||
sentTo = append(sentTo, names)
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = was })
|
||||
|
||||
long := time.Now().UTC().Add(-2 * time.Hour)
|
||||
stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee",
|
||||
Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Note: "a stopped at its first machine in tier 0: laptop refused what it was sent",
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &long, Commit: "c0ffee",
|
||||
First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}}
|
||||
if err := open.inventory.SavePlan(ctx, stopped); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A newer plan holding a refuses it: sending the older build would put it back.
|
||||
newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour),
|
||||
State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") {
|
||||
t.Fatalf("retried under a newer plan: %v", err)
|
||||
}
|
||||
newer.State = inventory.PlanSuperseded
|
||||
if err := open.inventory.SavePlan(ctx, newer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
said, err := retryPlan(ctx, open, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") {
|
||||
t.Fatalf("sent %v; said %q", sentTo, said)
|
||||
}
|
||||
p, err := open.inventory.PlanByID(ctx, stopped.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := p.Modules["a"]
|
||||
if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" {
|
||||
t.Fatalf("after retry the plan is %s, a %+v", p.State, a)
|
||||
}
|
||||
// And it goes on: a records (its policy sends nothing more), so the next tier is asked.
|
||||
advancePlans(ctx, open)
|
||||
if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" {
|
||||
t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"])
|
||||
}
|
||||
if len(*asked) != 1 {
|
||||
t.Fatalf("asked %v", *asked)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside
|
||||
// the plan, asked later, never answers it (novox/hq ADR 0219).
|
||||
func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := time.Now().UTC().Add(-time.Minute)
|
||||
plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked,
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}}
|
||||
if err := open.inventory.SavePlan(ctx, plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay")
|
||||
p, _ := open.inventory.PlanByID(ctx, plan.ID)
|
||||
if p.Modules["a"].State != "asked" {
|
||||
t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"])
|
||||
}
|
||||
// From the records too: a later build of the module recorded is not the plan's.
|
||||
recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}}
|
||||
if settleFromRecords(&p, p.Tiers[0], recorded, nil) {
|
||||
t.Fatalf("the records settled it with another build: %+v", p.Modules["a"])
|
||||
}
|
||||
planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own")
|
||||
if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" {
|
||||
t.Fatalf("its own build did not settle it: %+v", p.Modules["a"])
|
||||
}
|
||||
}
|
||||
|
||||
// rebuild of a build made at a commit asks what the module follows now, never the commit.
|
||||
func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
twoTiers(t, open)
|
||||
var refs []string
|
||||
was := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
refs = append(refs, ref)
|
||||
return was(c, source, path, ref)
|
||||
}
|
||||
if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a",
|
||||
Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 {
|
||||
t.Fatalf("asked %v at %v", *asked, refs)
|
||||
}
|
||||
}
|
||||
|
||||
// An ask the worker counts out that no machine said it started is cancelled only if no start comes
|
||||
// while a holder looks: one that does withdraws the cancel, and kill is what ends it.
|
||||
func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) {
|
||||
js, ask := aBuildQueue(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
was := holderLooks
|
||||
holderLooks = 300 * time.Millisecond
|
||||
t.Cleanup(func() { holderLooks = was })
|
||||
id := link.NewBuildID(time.Now())
|
||||
ask(id, "https://forge.example/novox/a.git")
|
||||
worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}})
|
||||
sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = sub.Unsubscribe() }()
|
||||
if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, _ := q.Find(id)
|
||||
if a.State != link.AskInFlight || a.On != "" {
|
||||
t.Fatalf("the taken ask reads %+v", a)
|
||||
}
|
||||
// The holder that took it says it started, while the cancel waits.
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)})
|
||||
_, _ = js.Context().Publish(link.BuildStarted(), started)
|
||||
}()
|
||||
if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") {
|
||||
t.Fatalf("a build that started was cancelled: %v", err)
|
||||
}
|
||||
if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled {
|
||||
t.Error("the withdrawn cancel is still marked")
|
||||
}
|
||||
if _, found, _ := open.inventory.BuildByID(ctx, id); found {
|
||||
t.Error("a withdrawn cancel recorded an outcome")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The same answers, in a shape something other than a person can read.
|
||||
//
|
||||
// A board reads through interfaces and holds nothing (novox/hq 03-DESIGN/01-to-be/11-a-board.md).
|
||||
// Everything it needs is already answered by these commands — as text, for people, which is not
|
||||
// something a page can read. So each of them can say it again as JSON.
|
||||
//
|
||||
// **`--json` rather than a serving API**, because nothing needs one yet: whatever serves a board
|
||||
// runs the command, and the constraint in the design holds either way — the board never touches a
|
||||
// context's store. An API is the larger thing and should wait until something is asking for it.
|
||||
//
|
||||
// **These shapes are hard to change once anything is built against them.** So they stay close to
|
||||
// what the domain already calls things, and carry no summary field that would have to be kept
|
||||
// true. Nothing here is derived that a reader could not derive.
|
||||
|
||||
// meshStatus is what `status --json` says: the three questions, in the order they are asked.
|
||||
type meshStatus struct {
|
||||
// Wrong is every machine whose last declaration was refused or partly failed.
|
||||
Wrong []machineDoing `json:"wrong"`
|
||||
// Quiet is every machine not heard from lately. Not the same as wrong: new, switched off and
|
||||
// unreachable are not "tried and could not".
|
||||
Quiet []machineQuiet `json:"quiet"`
|
||||
// Behind is every module built from something older than its source has.
|
||||
Behind []moduleBehind `json:"behind"`
|
||||
// Waiting is every machine not running what the mesh would send it. The same question as
|
||||
// Behind one level down: that says the catalogue is old, this says a machine is — and only
|
||||
// this one has somebody's change waiting inside it.
|
||||
Waiting []machineWaiting `json:"waiting"`
|
||||
// Reported is every machine's last word beside when it was last sent a declaration. A
|
||||
// machine whose report is newer than its send has acted on the current declaration; one
|
||||
// whose is older is still working — and Waiting cannot tell those apart, because the sent
|
||||
// digest is recorded at send, not at apply.
|
||||
Reported []machineReported `json:"reported"`
|
||||
// Plans is what the last merges produced and where each stands (novox/hq ADR 0162): the
|
||||
// open ones first, each saying its tier, what it waits for, and whether it has waited too long.
|
||||
Plans []planStatus `json:"plans"`
|
||||
// Unresolved is every machine that cannot be worked out at all, with what the mesh said when
|
||||
// it tried. **A machine here is in none of the lists above**: nothing was computed for it, so
|
||||
// there is nothing to compare it against and nothing it can be behind — which is why a
|
||||
// document without this field described a wholly blocked mesh as a well one.
|
||||
//
|
||||
// Per machine, and data. One node failing must never take the document away from a reader
|
||||
// asking about the others.
|
||||
Unresolved []machineUnresolved `json:"unresolved"`
|
||||
// Network is why the private network could not be computed, when it could not; absent when it
|
||||
// could. Almost always a consequence of Unresolved: a node that does not resolve is not on the
|
||||
// network, and a mesh whose hub is that node has no hub.
|
||||
Network string `json:"network,omitempty"`
|
||||
// Machines is how many the mesh knows about, so a reader can tell "none wrong" from
|
||||
// "none at all".
|
||||
Machines int `json:"machines"`
|
||||
// Adopted is every node still adopted (novox/hq ADR 0100); absent when none is.
|
||||
Adopted []string `json:"adopted,omitempty"`
|
||||
// Untaken is every module assigned to a machine that is holding what it found rather than
|
||||
// running what the module declares, because nothing took it (novox/hq 04-ISSUES/125). Absent
|
||||
// when nothing is held.
|
||||
//
|
||||
// **A document without this said an outage was a well mesh.** Read from what each machine
|
||||
// reported, so it is the machine's account and not the mesh's take-time listing.
|
||||
Untaken []machineUntaken `json:"untaken,omitempty"`
|
||||
// Filtered is every converged machine that is not filtered by the mesh alone (novox/hq ADR
|
||||
// 0168), one entry per rule set the mesh did not write — the found firewall in force again,
|
||||
// or a chain nobody speaks for. Absent when every converged machine is filtered by the mesh
|
||||
// alone. A document without this called a machine well while a predecessor's chain refused
|
||||
// what the mesh declared open.
|
||||
Filtered []machineFiltered `json:"filtered,omitempty"`
|
||||
// Unheld is every module on a machine whose resources are applied through a seat nothing on
|
||||
// that machine holds, with the modules that could hold it (novox/hq ADR 0207). Absent when every
|
||||
// dependency is met. Reported, not refused, until the switch.
|
||||
Unheld []catalogue.Unheld `json:"unheld,omitempty"`
|
||||
// Failing is every consumer a provider says it keeps failing, with the class of error, since
|
||||
// when, and when it was last said (novox/hq ADR 0224). Absent when no provider says so. A
|
||||
// document without this called the mesh well while the identity provider refused every consumer
|
||||
// for a day (04-ISSUES/179).
|
||||
Failing []inventory.ProviderStanding `json:"failing,omitempty"`
|
||||
}
|
||||
|
||||
// machineFiltered is one rule set on a converged machine that the mesh did not write and that
|
||||
// refuses traffic: where it is, whose the host reads it as, and what it refuses.
|
||||
type machineFiltered struct {
|
||||
Node string `json:"node"`
|
||||
Where string `json:"where"`
|
||||
Owner string `json:"owner"`
|
||||
Refuses string `json:"refuses"`
|
||||
}
|
||||
|
||||
// machineUntaken is one module a machine is holding rather than running, and how many resources of
|
||||
// it are held.
|
||||
type machineUntaken struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
// Held is how many of the module's resources the machine is keeping as it found them. Zero is
|
||||
// impossible here: a module with nothing held is not in this list.
|
||||
Held int `json:"held"`
|
||||
}
|
||||
|
||||
type machineUnresolved struct {
|
||||
Node string `json:"node"`
|
||||
// Problem is the mesh's own words, whole — newlines and all. It lists every requirement that
|
||||
// could not be met, and a first line alone would name one of them and hide the rest.
|
||||
Problem string `json:"problem"`
|
||||
}
|
||||
|
||||
type machineDoing struct {
|
||||
Node string `json:"node"`
|
||||
// Outcome is refused or failed. Kept distinct all the way out: they are fixed in different
|
||||
// places, and one word for both sends half the readers to the wrong one.
|
||||
Outcome string `json:"outcome"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Failed []struct {
|
||||
ID string `json:"id"`
|
||||
Error string `json:"error"`
|
||||
} `json:"failed,omitempty"`
|
||||
Applied int `json:"applied"`
|
||||
At time.Time `json:"at"`
|
||||
// Since is when this same failure was first reported and Times how many reports in a row
|
||||
// have said it; Stuck is the mesh's word for "enough of them" (novox/hq 04-ISSUES/065).
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
Times int `json:"times"`
|
||||
Stuck bool `json:"stuck"`
|
||||
}
|
||||
|
||||
type machineReported struct {
|
||||
Node string `json:"node"`
|
||||
Outcome string `json:"outcome"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Sent *time.Time `json:"sent,omitempty"`
|
||||
// Current is whether the last report names the declaration last sent. Not derivable from
|
||||
// the timestamps beside it: an apply begun under the previous declaration reports after the
|
||||
// next send, newer and still about the old words.
|
||||
Current bool `json:"current"`
|
||||
}
|
||||
|
||||
type machineWaiting struct {
|
||||
Node string `json:"node"`
|
||||
// Never is true when nothing has ever been sent to it. Not out of date: nobody has ever asked
|
||||
// this machine to be anything, and the two read differently to whoever is looking.
|
||||
Never bool `json:"never"`
|
||||
Sent *time.Time `json:"sent,omitempty"`
|
||||
}
|
||||
|
||||
type machineQuiet struct {
|
||||
Node string `json:"node"`
|
||||
// LastSeen is absent when the machine has never spoken, which is a different thing from
|
||||
// having been quiet for a while.
|
||||
LastSeen *time.Time `json:"lastSeen,omitempty"`
|
||||
}
|
||||
|
||||
type moduleBehind struct {
|
||||
Module string `json:"module"`
|
||||
BuiltFrom string `json:"builtFrom"`
|
||||
Head string `json:"head"`
|
||||
On []string `json:"on"`
|
||||
}
|
||||
|
||||
// statusAsJSON answers the same questions as the text form, from the same reading.
|
||||
//
|
||||
// **It takes the whole reading rather than a growing argument list**, which is what let a new
|
||||
// answer be added to the text form and forgotten here — the two are one function's output in two
|
||||
// shapes, and they must not be able to differ about what was asked.
|
||||
//
|
||||
// It never fails on account of the mesh. Every per-machine problem in here is a field, so one
|
||||
// machine that cannot be worked out cannot stop a caller reading about the others: a
|
||||
// machine-readable interface that stops being machine-readable exactly when something is wrong is
|
||||
// one nobody can build an alarm on.
|
||||
func statusAsJSON(asked answers) ([]byte, error) {
|
||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||
behind, sources := asked.behind, asked.sources
|
||||
waiting, reported := asked.waiting, asked.reported
|
||||
|
||||
out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{},
|
||||
Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{},
|
||||
Reported: []machineReported{}, Unresolved: []machineUnresolved{},
|
||||
Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused)}
|
||||
// In a stated order, so two readings of an unchanged mesh are the same document.
|
||||
untakenNodes := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
untakenNodes = append(untakenNodes, name)
|
||||
}
|
||||
sort.Strings(untakenNodes)
|
||||
for _, name := range untakenNodes {
|
||||
modules := make([]string, 0, len(asked.untaken[name]))
|
||||
for m := range asked.untaken[name] {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, m := range modules {
|
||||
out.Untaken = append(out.Untaken,
|
||||
machineUntaken{Node: name, Module: m, Held: asked.untaken[name][m]})
|
||||
}
|
||||
}
|
||||
filteredNodes := make([]string, 0, len(asked.filtered))
|
||||
for name := range asked.filtered {
|
||||
filteredNodes = append(filteredNodes, name)
|
||||
}
|
||||
sort.Strings(filteredNodes)
|
||||
for _, name := range filteredNodes {
|
||||
f := asked.filtered[name]
|
||||
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: "the found firewall",
|
||||
Owner: inventory.FilterFoundFirewall, Refuses: fw.Kind + " is in force again"})
|
||||
}
|
||||
for _, x := range f.Others() {
|
||||
out.Filtered = append(out.Filtered, machineFiltered{Node: name, Where: x.Where, Owner: x.Owner, Refuses: x.Refuses})
|
||||
}
|
||||
}
|
||||
out.Unheld = asked.unheld
|
||||
out.Failing = asked.failing
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
}
|
||||
sort.Slice(out.Unresolved, func(i, j int) bool {
|
||||
return out.Unresolved[i].Node < out.Unresolved[j].Node
|
||||
})
|
||||
for _, r := range reported {
|
||||
out.Reported = append(out.Reported, machineReported{
|
||||
Node: r.Node, Outcome: r.Outcome, At: r.At, Sent: r.Sent, Current: r.Current})
|
||||
}
|
||||
|
||||
for _, m := range waiting {
|
||||
out.Waiting = append(out.Waiting, machineWaiting{
|
||||
Node: m.Node, Never: m.Never, Sent: m.SentAt})
|
||||
}
|
||||
|
||||
for _, d := range wrong {
|
||||
row := machineDoing{
|
||||
Node: d.Node, Outcome: d.Outcome, Refused: d.Refused, Applied: d.Applied, At: d.At,
|
||||
Since: d.Since, Times: d.Times, Stuck: d.Stuck(),
|
||||
}
|
||||
for _, f := range d.Failed {
|
||||
row.Failed = append(row.Failed, struct {
|
||||
ID string `json:"id"`
|
||||
Error string `json:"error"`
|
||||
}{ID: f.ID, Error: f.Error})
|
||||
}
|
||||
out.Wrong = append(out.Wrong, row)
|
||||
}
|
||||
for _, n := range quiet {
|
||||
row := machineQuiet{Node: n.Name}
|
||||
if !n.LastSeen.IsZero() {
|
||||
seen := n.LastSeen
|
||||
row.LastSeen = &seen
|
||||
}
|
||||
out.Quiet = append(out.Quiet, row)
|
||||
}
|
||||
for module, on := range behind {
|
||||
from := sources[module]
|
||||
out.Behind = append(out.Behind, moduleBehind{
|
||||
Module: module, BuiltFrom: from.BuiltFrom, Head: from.Head, On: on,
|
||||
})
|
||||
}
|
||||
return json.MarshalIndent(out, "", " ")
|
||||
}
|
||||
|
||||
// say prints a value as JSON, for the commands that can answer either way.
|
||||
func say(value any) error {
|
||||
body, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The shape something other than a person reads.
|
||||
//
|
||||
// Hard to change once anything is built against it, so it stays close to what the domain already
|
||||
// calls things and carries no summary field that would have to be kept true.
|
||||
|
||||
func statusOf(t *testing.T, wrong []inventory.Doing, nodes, quiet []inventory.Node,
|
||||
behind map[string][]string, sources map[string]inventory.Source) map[string]any {
|
||||
t.Helper()
|
||||
body, err := statusAsJSON(answers{
|
||||
wrong: wrong, nodes: nodes, quiet: quiet, behind: behind, sources: sources})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
t.Fatalf("what a board would read is not JSON: %v", err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func TestRefusedAndFailedStayDistinctAllTheWayOut(t *testing.T) {
|
||||
// They are fixed in different places, so one word for both would send half the readers of a
|
||||
// page to the wrong one.
|
||||
got := statusOf(t,
|
||||
[]inventory.Doing{
|
||||
{Node: "one", Outcome: inventory.OutcomeRefused, Refused: "a file needs a path"},
|
||||
{Node: "two", Outcome: inventory.OutcomeFailed, Applied: 3,
|
||||
Failed: []inventory.FailedResource{{ID: "shell.pkg", Error: "target not found"}}},
|
||||
},
|
||||
[]inventory.Node{{Name: "one"}, {Name: "two"}}, nil, nil, nil)
|
||||
|
||||
wrong, _ := got["wrong"].([]any)
|
||||
if len(wrong) != 2 {
|
||||
t.Fatalf("got %v", got["wrong"])
|
||||
}
|
||||
first, _ := wrong[0].(map[string]any)
|
||||
if first["outcome"] != inventory.OutcomeRefused || first["refused"] == nil {
|
||||
t.Fatalf("a refusal did not survive: %v", first)
|
||||
}
|
||||
second, _ := wrong[1].(map[string]any)
|
||||
if second["outcome"] != inventory.OutcomeFailed {
|
||||
t.Fatalf("a failure did not survive: %v", second)
|
||||
}
|
||||
if second["applied"] != float64(3) {
|
||||
// "Three of eight" and "none of eight" are different machines.
|
||||
t.Fatalf("what did apply was not carried: %v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMachineThatNeverSpokeSaysSoByOmission(t *testing.T) {
|
||||
// Never heard from and quiet for a while are different situations, and a zero time would read
|
||||
// as a date in 1970 on any page that formatted it.
|
||||
got := statusOf(t, nil,
|
||||
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
|
||||
[]inventory.Node{{Name: "silent"}, {Name: "away", LastSeen: time.Now().Add(-3 * time.Hour)}},
|
||||
nil, nil)
|
||||
|
||||
quiet, _ := got["quiet"].([]any)
|
||||
if len(quiet) != 2 {
|
||||
t.Fatalf("got %v", got["quiet"])
|
||||
}
|
||||
never, _ := quiet[0].(map[string]any)
|
||||
if _, said := never["lastSeen"]; said {
|
||||
t.Fatalf("a machine that never spoke carries a time: %v", never)
|
||||
}
|
||||
away, _ := quiet[1].(map[string]any)
|
||||
if _, said := away["lastSeen"]; !said {
|
||||
t.Fatalf("a machine that has been quiet carries no time: %v", away)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingWrongIsAnEmptyListRatherThanNothing(t *testing.T) {
|
||||
// A page distinguishing "no machines are wrong" from "this field is missing" would have to
|
||||
// handle both, and null is the one that gets forgotten.
|
||||
got := statusOf(t, nil, []inventory.Node{{Name: "a", LastSeen: time.Now()}}, nil, nil, nil)
|
||||
for _, key := range []string{"wrong", "quiet", "behind"} {
|
||||
list, ok := got[key].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("%q is %T, not a list", key, got[key])
|
||||
}
|
||||
if len(list) != 0 {
|
||||
t.Fatalf("%q is not empty: %v", key, list)
|
||||
}
|
||||
}
|
||||
// And how many machines there are, so a reader can tell "none wrong" from "none at all".
|
||||
if got["machines"] != float64(1) {
|
||||
t.Fatalf("got %v", got["machines"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatIsBehindNamesTheMachinesRunningTheOldOne(t *testing.T) {
|
||||
// A module being out of date is a fact about the catalogue; machines running the old one is
|
||||
// the thing with consequences.
|
||||
got := statusOf(t, nil, nil, nil,
|
||||
map[string][]string{"shell": {"workstation", "laptop"}},
|
||||
map[string]inventory.Source{"shell": {BuiltFrom: "aaaaaaa1", Head: "bbbbbbb2"}})
|
||||
|
||||
behind, _ := got["behind"].([]any)
|
||||
if len(behind) != 1 {
|
||||
t.Fatalf("got %v", got["behind"])
|
||||
}
|
||||
row, _ := behind[0].(map[string]any)
|
||||
if row["module"] != "shell" || row["builtFrom"] != "aaaaaaa1" || row["head"] != "bbbbbbb2" {
|
||||
t.Fatalf("got %v", row)
|
||||
}
|
||||
on, _ := row["on"].([]any)
|
||||
if len(on) != 2 {
|
||||
t.Fatalf("the machines running the old one are not named: %v", row)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoSecretIsInWhatABoardReads(t *testing.T) {
|
||||
// Everything here comes from the mesh's own records, which hold no readable secret — but a
|
||||
// shape a page is built against is exactly where one would eventually be added for
|
||||
// convenience, so this says it out loud.
|
||||
body, err := statusAsJSON(answers{
|
||||
wrong: []inventory.Doing{{Node: "a", Outcome: inventory.OutcomeRefused,
|
||||
Refused: "resource \"x\": a file needs a path"}},
|
||||
nodes: []inventory.Node{{Name: "a"}},
|
||||
refused: map[string]string{"a": "nothing provides \"database\", wanted by web"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, word := range []string{"password", "secret", "sealed", "credential", "token"} {
|
||||
if strings.Contains(strings.ToLower(string(body)), word) {
|
||||
t.Fatalf("what a board reads carries a %q field:\n%s", word, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A failure that has been reported identically enough times is said to be stuck, with when it
|
||||
// began and how many times — so a board can tell a machine looping on something that will never
|
||||
// apply from one that failed a minute ago (novox/hq 04-ISSUES/065, ADR 0090).
|
||||
func TestAMachineFailingTheSameWayIsSaidToBeStuck(t *testing.T) {
|
||||
began := time.Date(2026, 9, 21, 9, 0, 0, 0, time.UTC)
|
||||
got := statusOf(t,
|
||||
[]inventory.Doing{
|
||||
{Node: "looping", Outcome: inventory.OutcomeFailed, Since: &began, Times: inventory.StuckAfter,
|
||||
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||
{Node: "once", Outcome: inventory.OutcomeFailed, Since: &began, Times: 1,
|
||||
Failed: []inventory.FailedResource{{ID: "img", Error: "no such image"}}},
|
||||
},
|
||||
[]inventory.Node{{Name: "looping"}, {Name: "once"}}, nil, nil, nil)
|
||||
|
||||
wrong, _ := got["wrong"].([]any)
|
||||
looping, _ := wrong[0].(map[string]any)
|
||||
if looping["stuck"] != true || looping["times"] != float64(inventory.StuckAfter) {
|
||||
t.Fatalf("three identical failures are stuck: %v", looping)
|
||||
}
|
||||
if since, _ := looping["since"].(string); !strings.HasPrefix(since, "2026-09-21T09:00:00") {
|
||||
t.Fatalf("when it began was not carried: %v", looping)
|
||||
}
|
||||
once, _ := wrong[1].(map[string]any)
|
||||
if once["stuck"] != false || once["times"] != float64(1) {
|
||||
t.Fatalf("one failure is not stuck: %v", once)
|
||||
}
|
||||
}
|
||||
|
||||
// A converged machine something other than the mesh filters is named, per rule set, and is not
|
||||
// well (novox/hq ADR 0168); one filtered by the mesh alone is not in the list.
|
||||
func TestAMachineNotFilteredByTheMeshAloneIsNamedAndNotWell(t *testing.T) {
|
||||
alone := inventory.Filtering{Filters: []inventory.Filter{
|
||||
{Where: "table inet mesh, chain input", Owner: inventory.FilterMesh, Refuses: "policy drop"},
|
||||
{Where: "table ip filter, chain DOCKER", Owner: inventory.FilterRuntime, Refuses: "drop"},
|
||||
{Where: "table ip filter, chain f2b-sshd", Owner: inventory.FilterBan, Refuses: "ip saddr 192.0.2.1 reject"},
|
||||
}}
|
||||
if !alone.Alone() {
|
||||
t.Fatal("the mesh's own, the runtime's and a ban are not the mesh alone")
|
||||
}
|
||||
notAlone := inventory.Filtering{
|
||||
Filters: append(alone.Filters, inventory.Filter{Where: "chain HAL-MESH-ONLY (iptables-legacy)",
|
||||
Owner: inventory.FilterOther, Refuses: `-A HAL-MESH-ONLY -m comment --comment "not public" -j DROP`}),
|
||||
FoundFirewall: &inventory.FoundFirewall{Kind: "ufw", Active: true},
|
||||
}
|
||||
asked := answers{nodes: []inventory.Node{{Name: "home-server"}, {Name: "laptop"}},
|
||||
filtered: map[string]inventory.Filtering{"home-server": notAlone}}
|
||||
if asked.well() {
|
||||
t.Fatal("a machine not filtered by the mesh alone reads as well")
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed struct {
|
||||
Filtered []map[string]string `json:"filtered"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(parsed.Filtered) != 2 {
|
||||
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||
}
|
||||
if parsed.Filtered[0]["node"] != "home-server" || parsed.Filtered[0]["owner"] != inventory.FilterFoundFirewall ||
|
||||
parsed.Filtered[1]["where"] != "chain HAL-MESH-ONLY (iptables-legacy)" || parsed.Filtered[1]["owner"] != inventory.FilterOther {
|
||||
t.Fatalf("filtered: %v", parsed.Filtered)
|
||||
}
|
||||
if body, _ := statusAsJSON(answers{nodes: asked.nodes}); strings.Contains(string(body), `"filtered"`) {
|
||||
t.Fatal("a mesh filtered by itself alone carries a filtered list")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a build is recorded as, and what it is announced and handed on as.
|
||||
//
|
||||
// **Recorded by what it is; routed where it is used** (novox/hq 04-ISSUES/102). The builder
|
||||
// announces each artifact by the reference it pushed — `<registry>:<port>/<module>/<artifact>@sha256:…`
|
||||
// — and the manifest with those references in it. The mesh records the digest and the path
|
||||
// (catalogue.Recorded) and composes the store's address back in wherever a machine or a builder
|
||||
// needs a reference it can fetch: a declaration, a replay to the catalogue, the bases a build is
|
||||
// given. Nothing recorded carries a port, so moving the store is a settings change and not a
|
||||
// rebuild of everything the mesh has ever built.
|
||||
|
||||
// recordedManifest is a build's manifest with the store's address taken off every reference the
|
||||
// build itself made. Other references — an image a module runs from a public registry — are what
|
||||
// they were, which is why this rewrites only what `made` names rather than everything that looks
|
||||
// like an address.
|
||||
func recordedManifest(raw json.RawMessage, made []inventory.Artifact) json.RawMessage {
|
||||
announced := map[string]bool{}
|
||||
for _, a := range made {
|
||||
announced[a.Reference] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !announced[ref] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Recorded(ref), true
|
||||
})
|
||||
}
|
||||
|
||||
// routedManifest is a recorded manifest with the store's address, as this network reaches it now,
|
||||
// composed into every reference the build made — recorded either way, before or after references
|
||||
// were kept without their address.
|
||||
func routedManifest(raw json.RawMessage, made []inventory.Artifact, address string) json.RawMessage {
|
||||
recorded := map[string]bool{}
|
||||
for _, a := range made {
|
||||
recorded[catalogue.Recorded(a.Reference)] = true
|
||||
}
|
||||
return withReferences(raw, func(ref string) (string, bool) {
|
||||
if !recorded[catalogue.Recorded(ref)] {
|
||||
return ref, false
|
||||
}
|
||||
return catalogue.Rerouted(ref, address), true
|
||||
})
|
||||
}
|
||||
|
||||
// withReferences applies `rewrite` to each resource's `image` and `source`, and hands the manifest
|
||||
// back untouched — byte for byte — when nothing changed or it could not be read: what a manifest
|
||||
// is, is the parser's to say, and it says so with a better sentence than anything here would.
|
||||
func withReferences(raw json.RawMessage, rewrite func(string) (string, bool)) json.RawMessage {
|
||||
if len(raw) == 0 {
|
||||
return raw
|
||||
}
|
||||
var manifest map[string]any
|
||||
if err := json.Unmarshal(raw, &manifest); err != nil {
|
||||
return raw
|
||||
}
|
||||
resources, _ := manifest["resources"].([]any)
|
||||
changed := false
|
||||
for _, r := range resources {
|
||||
resource, ok := r.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, key := range []string{"image", "source"} {
|
||||
if written, ok := resource[key].(string); ok {
|
||||
if rewritten, did := rewrite(written); did && rewritten != written {
|
||||
resource[key] = rewritten
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return raw
|
||||
}
|
||||
out, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// routedArtifacts is a build's artifacts as something can fetch them now.
|
||||
func routedArtifacts(made []inventory.Artifact, address string) []inventory.Artifact {
|
||||
if address == "" {
|
||||
return made
|
||||
}
|
||||
out := make([]inventory.Artifact, 0, len(made))
|
||||
for _, a := range made {
|
||||
out = append(out, inventory.Artifact{Name: a.Name, Kind: a.Kind,
|
||||
Reference: catalogue.Rerouted(a.Reference, address)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// whereTheStoreIs is the artifact store's address as something on `forNode` reaches it, or "" —
|
||||
// read for a caller that has the inventory open and nothing else in hand. With no node named, the
|
||||
// store's own node: loopback when nothing is on the network yet.
|
||||
func whereTheStoreIs(ctx context.Context, inv *inventory.Inventory, forNode string) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if forNode == "" {
|
||||
if holder, _, found, err := artifactStoreHolder(ctx, inv); err != nil {
|
||||
return "", err
|
||||
} else if found {
|
||||
forNode = holder
|
||||
}
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, forNode)
|
||||
}
|
||||
|
||||
// whereABuilderReachesTheStore is the store's address for the machine that builds: the network's
|
||||
// when there is one, else loopback on the store's own node — when that node also holds a module
|
||||
// requiring the store, which is what a builder is (genesis: one node holds both).
|
||||
func whereABuilderReachesTheStore(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
holder, _, found, err := artifactStoreHolder(ctx, inv)
|
||||
if err != nil || !found {
|
||||
return "", err
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, holder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
besideIt := false
|
||||
for _, a := range assigned {
|
||||
for _, r := range shelf[a].Requires {
|
||||
if r == catalogue.ArtifactStoreProvision {
|
||||
besideIt = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !besideIt {
|
||||
holder = ""
|
||||
}
|
||||
return artifactStoreAddress(ctx, inv, shelf, holder)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,214 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it,
|
||||
// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart.
|
||||
func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) {
|
||||
edges := []inventory.Edge{
|
||||
// build dependencies: images on the runtime, a plugin on one of them
|
||||
{From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
{From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
{From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared},
|
||||
// a code dependency: the proxy packages the controller's source — same tier
|
||||
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
// runtime dependencies: everything source-built is built by the builder
|
||||
{From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn},
|
||||
{From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn},
|
||||
}
|
||||
// The runtime image moved: everything on it, and what is built by what is on it.
|
||||
set := reachableFrom([]string{"mesh-tools"}, edges)
|
||||
// What stands on the runtime, and the builder that stands on it; not the controller, which the
|
||||
// builder merely builds, nor the proxy that packages the controller.
|
||||
want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"}
|
||||
if len(set) != len(want) {
|
||||
t.Fatalf("reachable from the runtime: %v, want %v", set, want)
|
||||
}
|
||||
tiers := tiersOf(set, edges)
|
||||
pos := map[string]int{}
|
||||
for i, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
pos[m] = i
|
||||
}
|
||||
}
|
||||
if pos["mesh-tools"] != 0 || pos["builder"] != 1 {
|
||||
t.Fatalf("the runtime then the builder: %v", tiers)
|
||||
}
|
||||
if !(pos["shop"] > pos["builder"] && pos["postgres"] > pos["builder"]) {
|
||||
t.Fatalf("what the builder builds comes after the builder: %v", tiers)
|
||||
}
|
||||
if pos["shop-plugin"] <= pos["shop"] {
|
||||
t.Fatalf("a plugin after what it is declared on: %v", tiers)
|
||||
}
|
||||
if hasCycle(tiers, edges) {
|
||||
t.Fatalf("no cycle here: %v", tiers)
|
||||
}
|
||||
|
||||
// The controller alone moved: the proxy with it, nothing else.
|
||||
small := reachableFrom([]string{"mesh-controller"}, edges)
|
||||
if len(small) != 3 {
|
||||
t.Fatalf("a controller merge rebuilds the controller and what packages it: %v", small)
|
||||
}
|
||||
// The builder packages the controller's source (same tier by that edge) and the controller is
|
||||
// built by the builder (next tier by that one): the builder first, then the controller and the
|
||||
// proxy together — a code dependency in one tier, a runtime dependency across tiers.
|
||||
smallTiers := tiersOf(small, edges)
|
||||
if len(smallTiers) != 2 || smallTiers[0][0] != "builder" || len(smallTiers[1]) != 2 {
|
||||
t.Fatalf("the builder, then the controller and the proxy together: %v", smallTiers)
|
||||
}
|
||||
// The builder alone moved: the builder, and nothing it builds.
|
||||
if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 {
|
||||
t.Fatalf("a build machine change rebuilds the build machine alone: %v", only)
|
||||
}
|
||||
|
||||
// Only a runtime dependency gates on deployment, and only when the module rolls out.
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges)
|
||||
p.Tier = pos["builder"]
|
||||
rollsOut := func(m string) bool { return m == "builder" }
|
||||
if g := gates(p, edges, rollsOut); len(g) != 1 || g[0] != "builder" {
|
||||
t.Fatalf("the builder gates the tier after it: %v", g)
|
||||
}
|
||||
p.Tier = 0
|
||||
if g := gates(p, edges, rollsOut); len(g) != 0 {
|
||||
t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g)
|
||||
}
|
||||
if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 {
|
||||
t.Fatalf("a module that only records its upgrade gates nothing: %v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A gate is open once every machine running the module has reported after it was built.
|
||||
func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) {
|
||||
built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC)
|
||||
before, after := built.Add(-time.Minute), built.Add(time.Minute)
|
||||
reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}}
|
||||
ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports)
|
||||
if ok || len(waiting) != 1 || waiting[0] != "home-server" {
|
||||
t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting)
|
||||
}
|
||||
if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok {
|
||||
t.Fatal("the machine that reported after the build holds the gate open")
|
||||
}
|
||||
if ok, _ := applied("builder", built, nil, reports); !ok {
|
||||
t.Fatal("a module running nowhere gates nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// A cycle is not lost: what remains is one last tier, and the caller says so.
|
||||
func TestACycleIsOneLastTierAndSaidSo(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}}
|
||||
tiers := tiersOf([]string{"a", "b"}, edges)
|
||||
if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) {
|
||||
t.Fatalf("a cycle should be one tier of two, said: %v", tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the
|
||||
// bundle a tier after the toolchain, not beside it.
|
||||
func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) {
|
||||
edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}}
|
||||
p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"},
|
||||
[]string{"mesh-tools", "node-tools"}, edges)
|
||||
if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" {
|
||||
t.Fatalf("the toolchain, then the bundle: %v", p.Tiers)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it —
|
||||
// the controller rebuilding itself — settles from the build records instead of waiting for ever.
|
||||
func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) {
|
||||
asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"mesh-controller": {State: "asked", AskedAt: &asked},
|
||||
"builder": {State: "asked", AskedAt: &asked},
|
||||
}}
|
||||
records := map[string][]inventory.Build{
|
||||
// Newest first, as Builds answers: the build after the ask is the outcome.
|
||||
"mesh-controller": {
|
||||
{ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)},
|
||||
{ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)},
|
||||
},
|
||||
// Only a build from before the ask: not this ask's outcome.
|
||||
"builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}},
|
||||
}
|
||||
if !settleFromRecords(&p, p.Tiers[0], records, nil) {
|
||||
t.Fatal("nothing settled, though the controller's build is recorded after the ask")
|
||||
}
|
||||
if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil {
|
||||
t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s)
|
||||
}
|
||||
if s := p.Modules["builder"]; s.State != "asked" {
|
||||
t.Errorf("an ask with no record after it was settled: %+v", s)
|
||||
}
|
||||
|
||||
// novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier
|
||||
// plan's late outcome — is not this ask's, built or failed.
|
||||
r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}},
|
||||
Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}}
|
||||
late := map[string][]inventory.Build{"postgres": {
|
||||
{ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)},
|
||||
}}
|
||||
if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" {
|
||||
t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"])
|
||||
}
|
||||
// Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it.
|
||||
late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73",
|
||||
Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)})
|
||||
if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" ||
|
||||
r.Modules["postgres"].Commit != "4bcd5f73" {
|
||||
t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"])
|
||||
}
|
||||
|
||||
// A failure recorded after the ask fails the plan, as hearing it would have.
|
||||
q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}},
|
||||
Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}}
|
||||
settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil)
|
||||
if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" {
|
||||
t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"])
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine's report, made between the send to it and the send to the rest, opens the gate for it:
|
||||
// each machine is judged from its own send, not from the last one (novox/hq issue 256).
|
||||
func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) {
|
||||
built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC)
|
||||
firstSent := built.Add(31 * time.Second)
|
||||
firstReported := built.Add(43 * time.Second)
|
||||
restSent := built.Add(58 * time.Second)
|
||||
restReported := built.Add(74 * time.Second)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &firstReported},
|
||||
{Node: "g14", At: &restReported},
|
||||
}
|
||||
since := func(node string) time.Time {
|
||||
if node == "ace" {
|
||||
return firstSent
|
||||
}
|
||||
return restSent
|
||||
}
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok {
|
||||
t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting)
|
||||
}
|
||||
// The old reading, every machine from the last send, is what held the plan.
|
||||
if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok {
|
||||
t.Fatal("the single-moment reading should hold the first machine back")
|
||||
}
|
||||
early := built.Add(10 * time.Second)
|
||||
if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" {
|
||||
t.Fatal("a report from before the machine was sent opened the gate")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// Moving the mesh's own traffic to the bus being built (novox/hq ADR 0116 step 5).
|
||||
//
|
||||
// **The whole mesh moves at once, so there is nothing to inspect afterwards.** Every seam ships both
|
||||
// transports and every one of them chooses by a single fact; this is the step that flips it. That
|
||||
// shape is deliberate — steps 1 to 4 leave every node where it is, so the cost of being wrong stays
|
||||
// bounded until here — and it means the useful work is almost all in the checking.
|
||||
//
|
||||
// So `rollout check` is the command that matters and the one that can be run any number of times
|
||||
// against a mesh that is serving. It answers from records: what is missing, and what would happen.
|
||||
// `rollout` itself refuses unless the check is clean.
|
||||
//
|
||||
// **The old broker goes with the move, and goes last** (novox/hq ADR 0131): AMQP is not a provision,
|
||||
// so once every machine reports on the new bus its module is unassigned. Only the mesh's own traffic
|
||||
// is what moves, which is why this is survivable at all: what breaks if it goes wrong is the mesh's
|
||||
// ability to change things, not the services its modules are serving — measured on 2026-09-27, when
|
||||
// a seat emptied mid-change and the control plane looped for two hours while every service stayed up.
|
||||
|
||||
const rolloutUsage = "rollout check | rollout mint [--again] | rollout hand <node> | rollout --confirm"
|
||||
|
||||
func rolloutCommand(ctx context.Context, args []string) error {
|
||||
switch {
|
||||
case len(args) == 1 && args[0] == "check":
|
||||
return rolloutCheck(ctx)
|
||||
case len(args) == 1 && args[0] == "mint":
|
||||
return rolloutMint(ctx, false)
|
||||
case len(args) == 2 && args[0] == "hand":
|
||||
return rolloutHand(ctx, args[1])
|
||||
case len(args) == 2 && args[0] == "mint" && args[1] == "--again":
|
||||
// Every credential minted afresh, whether or not one exists — for a mint that was wrong
|
||||
// before anything was pushed. Afterwards nothing that received the old one still works,
|
||||
// which is fine exactly when nothing received it.
|
||||
return rolloutMint(ctx, true)
|
||||
case len(args) == 1 && args[0] == "--confirm":
|
||||
return errors.New(
|
||||
"the rollout itself is not built yet: `rollout check` answers whether it could run, and " +
|
||||
"what is missing. Moving every node at once is the one step with nothing to inspect " +
|
||||
"afterwards, so it is not being written before the check it depends on has been run " +
|
||||
"against a real mesh")
|
||||
default:
|
||||
return errors.New(rolloutUsage)
|
||||
}
|
||||
}
|
||||
|
||||
// rolloutCheck says whether the mesh could move, and what would happen if it did.
|
||||
func rolloutCheck(ctx context.Context) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
state, err := readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println("the bus this mesh would move to")
|
||||
if state.TheBus == "" {
|
||||
fmt.Printf(" nothing names one (%s is unset)\n", broker.NATSVar)
|
||||
} else {
|
||||
standing := "not answering"
|
||||
if state.ServerStanding {
|
||||
standing = "answering"
|
||||
}
|
||||
fmt.Printf(" %s — %s\n", state.TheBus, standing)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
fmt.Println("what would move")
|
||||
for _, step := range broker.WhatMoves(state) {
|
||||
fmt.Printf(" %s\n", step)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
why := notReadyOf(state)
|
||||
if len(why) == 0 {
|
||||
fmt.Println("nothing is missing: this mesh could move its bus.")
|
||||
fmt.Println()
|
||||
fmt.Println("Read `what would move` above once more before running it. Every node moves at the")
|
||||
fmt.Println("same moment and there is no half-moved state to look at afterwards.")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("not ready — %d thing(s) to do first:\n", len(why))
|
||||
for i, w := range why {
|
||||
fmt.Printf(" %d. %s\n", i+1, w)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// readinessOf gathers what the mesh knows about its own ability to move.
|
||||
//
|
||||
// Reads and one dial, and nothing is written. Safe to run on a mesh that is serving, which is the
|
||||
// point: the answer is only useful if it can be had without committing to anything.
|
||||
func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readiness, error) {
|
||||
state := broker.Readiness{
|
||||
Credentialled: map[string]bool{},
|
||||
ModuleCredentialled: map[string]bool{},
|
||||
// The old broker keeps its other clients on this installation, and saying so is how the plan
|
||||
// stops reading as a retirement.
|
||||
}
|
||||
|
||||
address, _, err := broker.OnNATS()
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
state.TheBus = address
|
||||
if address != "" {
|
||||
// One dial, briefly. "Is it answering" is the one fact records cannot hold, and a mesh about
|
||||
// to move onto a server that is not there should hear it here rather than afterwards.
|
||||
//
|
||||
// **Dialled the way the mesh dials it** — credential and pin — because a bare connect to a
|
||||
// bus that requires TLS and a user fails at the handshake, and the check then reported a
|
||||
// standing server as absent (seen live, 2026-09-28).
|
||||
if js, err := broker.Dial(address, nats.Timeout(5*time.Second)); err == nil {
|
||||
state.ServerStanding = true
|
||||
js.Close()
|
||||
}
|
||||
}
|
||||
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
state.Nodes = append(state.Nodes, n.Name)
|
||||
_, has := kept[broker.Principal{Kind: broker.KindNode, Node: n.Name}.Username()]
|
||||
state.Credentialled[n.Name] = has
|
||||
|
||||
assigned, err := inv.Assigned(ctx, n.Name)
|
||||
if err != nil {
|
||||
return state, err
|
||||
}
|
||||
for _, module := range assigned {
|
||||
m, known := shelf[module]
|
||||
if !known {
|
||||
continue
|
||||
}
|
||||
// The machine that holds the bus seat is the one that would be sent the user list.
|
||||
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
|
||||
state.Holder = n.Name
|
||||
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
|
||||
}
|
||||
// A module that never speaks needs no credential, so it is not counted as missing one.
|
||||
if !speaksOnTheBus(m) {
|
||||
continue
|
||||
}
|
||||
named := n.Name + "/" + module
|
||||
state.Modules = append(state.Modules, named)
|
||||
_, hasOne := kept[broker.Principal{
|
||||
Kind: broker.KindModule, Node: n.Name, Module: module,
|
||||
}.Username()]
|
||||
state.ModuleCredentialled[named] = hasOne
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// speaksOnTheBus says whether a module reaches the bus at all.
|
||||
//
|
||||
// A third of the catalogue never does (novox/hq ADR 0120), and counting those as missing a credential
|
||||
// would bury the ones that matter under a list nobody can act on.
|
||||
func speaksOnTheBus(m catalogue.Manifest) bool {
|
||||
return len(m.EmitsAll()) > 0 || len(m.Consumes) > 0 || len(m.Tools) > 0 ||
|
||||
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
|
||||
}
|
||||
|
||||
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
|
||||
// list became part of one.
|
||||
//
|
||||
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
|
||||
// has still been sent it, and this question is about whether the mesh did its part.
|
||||
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
|
||||
digest, err := inv.Outstanding(ctx, node)
|
||||
return err == nil && strings.TrimSpace(digest) != ""
|
||||
}
|
||||
|
||||
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
|
||||
// printing. The reasoning itself is the broker package's, where it is pure.
|
||||
func notReadyOf(state broker.Readiness) []string { return broker.NotReady(state) }
|
||||
|
||||
// rolloutMint gives every principal the new bus will have a credential it does not yet have, and
|
||||
// puts each where its owner reads it (novox/hq design 28, task 5.2): a machine's as a membership
|
||||
// sealed into its declaration, a module's as its broker secret, the control plane's own as its
|
||||
// `bus` secret. Idempotent: what already has a hash is left alone, so running it again is harmless.
|
||||
//
|
||||
// **Before anything moves, and it is what makes moving possible.** A machine moved without a
|
||||
// credential cannot come back, and afterwards there is no bus to tell it anything over — which is
|
||||
// why `rollout check` refuses until this has run. The bus's address is worked out here, from where
|
||||
// the module that provides it is assigned, rather than read from this process's environment: this
|
||||
// process is still on the old bus when this runs, and must be.
|
||||
func rolloutMint(ctx context.Context, again bool) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bus's certificate is not known to this process, and every membership "+
|
||||
"must carry its fingerprint: %w", err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var busNode, controllerNode string
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case e.Manifest.ClaimsSeat("mesh-broker") && providesBus(e.Manifest) && len(e.On) > 0:
|
||||
busNode = e.On[0]
|
||||
case e.Manifest.Module == "mesh-controller" && len(e.On) > 0:
|
||||
controllerNode = e.On[0]
|
||||
}
|
||||
}
|
||||
if busNode == "" {
|
||||
return errors.New("no assigned module provides mesh-bus and claims mesh-broker, so there is no " +
|
||||
"bus to mint credentials for — register and assign it first")
|
||||
}
|
||||
onNetwork, err := whereEveryoneIs(ctx, inv, shelf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
busHost := onNetwork[busNode]
|
||||
if busHost == "" {
|
||||
// **The hub is not in that map.** The machine that took over the tunnel is where the current
|
||||
// bus already answers, and every machine dials it at the address the mesh handed them — so
|
||||
// when the new bus runs on the same machine, that address is the one to tell them, with the
|
||||
// new port. Found live: the control node is the hub, and the map lists the machines placed
|
||||
// around it.
|
||||
// The host alone: no scheme (BareAddress adds one where none was, which is the wrong
|
||||
// direction here — every URL built below adds its own) and no port.
|
||||
_, _, host := broker.CredentialIn(known.Address)
|
||||
if host == "" {
|
||||
host = known.Address
|
||||
}
|
||||
if _, after, hasScheme := strings.Cut(host, "://"); hasScheme {
|
||||
host = after
|
||||
}
|
||||
host = strings.TrimSpace(host)
|
||||
if i := strings.LastIndex(host, ":"); i > 0 && !strings.Contains(host[i:], "]") {
|
||||
host = host[:i]
|
||||
}
|
||||
if host == "" {
|
||||
return fmt.Errorf("%s runs the new bus and has no address on the private network, and the "+
|
||||
"current bus's address is unknown too, so no machine could be told where it is", busNode)
|
||||
}
|
||||
busHost = host
|
||||
}
|
||||
busAddress := busHost + ":4222"
|
||||
|
||||
records, err := inv.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
kept, err := inv.BusUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hashes := make(map[string]string, len(kept))
|
||||
for name, u := range kept {
|
||||
hashes[name] = u.PasswordHash
|
||||
}
|
||||
_, missing := broker.WithPasswords(users, hashes)
|
||||
wanted := map[string]bool{}
|
||||
for _, m := range missing {
|
||||
wanted[m] = true
|
||||
}
|
||||
|
||||
var machines, modules, skipped int
|
||||
for _, p := range users {
|
||||
if !again && !wanted[p.Username()] {
|
||||
continue
|
||||
}
|
||||
switch p.Kind {
|
||||
case broker.KindController:
|
||||
if controllerNode == "" {
|
||||
return errors.New("the control plane is not assigned anywhere, so its credential has nowhere to go")
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusController})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
url := "nats://" + p.Username() + ":" + password + "@" + busAddress
|
||||
if err := inv.AcceptSecretForModule(ctx, controllerNode, "mesh-controller", "bus", url); err != nil {
|
||||
return fmt.Errorf("the control plane's credential is minted and could not be sealed to %s: %w", controllerNode, err)
|
||||
}
|
||||
fmt.Printf("control plane: credential minted, sealed to %s as its `bus` secret\n", controllerNode)
|
||||
|
||||
case broker.KindNode:
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: p.Node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
membership, _ := json.Marshal(map[string]string{
|
||||
"broker": busAddress, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||
})
|
||||
key, err := inv.SealingKeyOf(ctx, p.Node)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s has no sealing key, so its membership cannot be sealed to it: %w", p.Node, err)
|
||||
}
|
||||
sealed, err := secrets.Seal(key, membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.PutBusMembership(ctx, p.Node, sealed); err != nil {
|
||||
return err
|
||||
}
|
||||
machines++
|
||||
|
||||
case broker.KindModule, broker.KindNodeTools:
|
||||
// The runtime is minted and delivered exactly as a module is (novox/hq ADR 0175): it is
|
||||
// issued as the module it stands for, to that module's `broker` secret.
|
||||
if p.Module == "mesh-controller" {
|
||||
// The control plane is a module too, and its `broker` secret is the old bus's
|
||||
// credential it is still using while this runs. Writing the new bus's blob there
|
||||
// cut the mesh off from its own old bus mid-move (2026-09-28). Its new-bus credential
|
||||
// is the controller principal's `bus` secret above; nothing else is needed here.
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
m, inShelf := shelf[p.Module]
|
||||
if !inShelf {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads {
|
||||
fmt.Printf(" %s on %s speaks on the bus but declares no `broker` secret to receive a credential in; skipped\n", p.Module, p.Node)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: busKindOf(p.Module), Node: p.Node, Module: p.Module})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := issueWith(ctx, inv, m, p.Node, "", known, busAddress, p.Username(), password); err != nil {
|
||||
return err
|
||||
}
|
||||
modules++
|
||||
|
||||
default:
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
fmt.Printf("minted for %d machine(s) and %d module runtime(s); %d skipped; the bus is at %s\n",
|
||||
machines, modules, skipped, busAddress)
|
||||
fmt.Println(" each machine's membership and each module's credential arrive with the next push of its machine;")
|
||||
fmt.Println(" push the machine running the bus first, so the bus stands with its user list before anything dials it")
|
||||
return nil
|
||||
}
|
||||
|
||||
// providesBus is whether a manifest provides the mesh's bus.
|
||||
func providesBus(m catalogue.Manifest) bool {
|
||||
for _, o := range m.Provides {
|
||||
if o.Name == "mesh-bus" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// rolloutHand mints a machine its credential for the new bus afresh and prints its membership
|
||||
// once, for an operator to carry by hand — the rescue for a machine that cannot be reached over
|
||||
// any bus: rotated while it still held the old password, or reachable only by ssh. The plaintext
|
||||
// exists on this terminal and then only where it is written; the store keeps the hash, and the
|
||||
// sealed copy in the machine's declaration is replaced too, so the next push says the same.
|
||||
func rolloutHand(ctx context.Context, node string) error {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
known, err := broker.FromEnvironment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bus's certificate is not known to this process: %w", err)
|
||||
}
|
||||
busAddress, _, err := broker.OnNATS()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if busAddress == "" {
|
||||
return errors.New("this control plane is not on the new bus, so there is no membership to hand out")
|
||||
}
|
||||
_, _, bare := broker.CredentialIn(busAddress)
|
||||
if _, after, has := strings.Cut(bare, "://"); has {
|
||||
bare = after
|
||||
}
|
||||
if _, err := inv.NodeByName(ctx, node); err != nil {
|
||||
return err
|
||||
}
|
||||
p := broker.Principal{Kind: broker.KindNode, Node: node}
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.Username(), Kind: inventory.BusNode, Node: node})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
membership, _ := json.Marshal(map[string]string{
|
||||
"broker": bare, "fingerprint": known.Fingerprint, "password": password, "transport": "nats",
|
||||
})
|
||||
key, err := inv.SealingKeyOf(ctx, node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sealed, err := secrets.Seal(key, membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.PutBusMembership(ctx, node, sealed); err != nil {
|
||||
return err
|
||||
}
|
||||
// The one line of output is the membership itself, so it can be piped to the machine without
|
||||
// being read on the way. Everything else goes to stderr.
|
||||
fmt.Fprintf(os.Stderr, "%s's credential is minted afresh. Write this to %s on it and restart its host; "+
|
||||
"then push the machine running the bus so the user list carries the new hash.\n",
|
||||
node, catalogue.BusMembershipPath)
|
||||
fmt.Println(string(membership))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only
|
||||
// once that machine has reported it applied; a module whose policy says together goes everywhere at
|
||||
// once, as before.
|
||||
func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) {
|
||||
running := []string{"novox", "ace", "g14"}
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
now := sentAt.Add(5 * time.Minute)
|
||||
bound := 30 * time.Minute
|
||||
after := sentAt.Add(time.Minute)
|
||||
next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep {
|
||||
return nextRollout(s, running, together, reports, now, bound)
|
||||
}
|
||||
|
||||
// Together: every machine at once.
|
||||
if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first {
|
||||
t.Fatalf("a together policy did not send every machine at once: %+v", step)
|
||||
}
|
||||
|
||||
// Otherwise the first by name, alone, when none has reported lately.
|
||||
step := next(inventory.PlanModule{}, running, false, nil)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) {
|
||||
t.Fatalf("the first send was %+v, wanted ace alone", step)
|
||||
}
|
||||
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
report := func(outcome string, current bool) []inventory.Reported {
|
||||
return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current},
|
||||
{Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}}
|
||||
}
|
||||
|
||||
// No report yet, or one about an older declaration than it was last sent: wait.
|
||||
for what, reports := range map[string][]inventory.Reported{
|
||||
"no report": nil,
|
||||
"a report about older": report(inventory.OutcomeApplied, false),
|
||||
} {
|
||||
step := next(state, running, false, reports)
|
||||
if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" {
|
||||
t.Errorf("%s: %+v, wanted to wait for ace", what, step)
|
||||
}
|
||||
}
|
||||
|
||||
// Applied what it was last sent: the rest, and only the rest.
|
||||
step = next(state, running, false, report(inventory.OutcomeApplied, true))
|
||||
if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) {
|
||||
t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step)
|
||||
}
|
||||
|
||||
// Failed or refused: stop, the rest untouched.
|
||||
for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} {
|
||||
step := next(state, running, false, report(outcome, true))
|
||||
if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) ||
|
||||
!reflect.DeepEqual(step.rest, []string{"novox", "g14"}) {
|
||||
t.Errorf("a first machine that %s it: %+v", outcome, step)
|
||||
}
|
||||
}
|
||||
|
||||
// The machine holding the bus went with the first send: the rest wait for it too, and it is
|
||||
// not sent again.
|
||||
both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt}
|
||||
half := report(inventory.OutcomeApplied, true)
|
||||
if step := next(both, running, false, half); step.waiting != "novox" {
|
||||
t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step)
|
||||
}
|
||||
all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true})
|
||||
if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step)
|
||||
}
|
||||
|
||||
// One machine, or none: nothing is waited for that cannot come.
|
||||
if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first {
|
||||
t.Fatalf("a module nothing runs was sent: %+v", step)
|
||||
}
|
||||
if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" {
|
||||
t.Fatalf("a module on one machine waited for more: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there,
|
||||
// naming the machine and the bound; the rest are left alone.
|
||||
func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) {
|
||||
sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt}
|
||||
step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute)
|
||||
if !strings.Contains(step.failed, "ace did not report it applied within 30m") ||
|
||||
!reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 {
|
||||
t.Fatalf("a silent first machine: %+v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The first machine is the first by name among those heard from lately: a laptop that is away is
|
||||
// not the one the rest wait on. When none has been heard from, the first by name.
|
||||
func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) {
|
||||
now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour)
|
||||
reports := []inventory.Reported{
|
||||
{Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately},
|
||||
}
|
||||
step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute)
|
||||
if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) {
|
||||
t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step)
|
||||
}
|
||||
}
|
||||
|
||||
// An announced move of a module an open plan is still rolling out is left to the plan: sending it
|
||||
// here as well put the bundle on every machine at once (novox/hq issue 249).
|
||||
func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) {
|
||||
sent := time.Now()
|
||||
plans := []inventory.Plan{
|
||||
{ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}},
|
||||
{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}},
|
||||
}
|
||||
if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" {
|
||||
t.Fatalf("a module the plan is rolling out was not left to it: %q", got)
|
||||
}
|
||||
for _, m := range []string{"gitea", "keycloak"} {
|
||||
if got := rolledOutByAPlan(plans, m); got != "" {
|
||||
t.Errorf("%s, which no plan will send, was left to %s", m, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A plan that ends without sending what it built says so, with the remedy; a module whose rollout
|
||||
// stopped at its first machine is not among them.
|
||||
func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) {
|
||||
at := time.Now()
|
||||
p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1",
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"agent": {State: "built"},
|
||||
"stopped": {State: "built", First: []string{"ace"}, FirstAt: &at},
|
||||
"sent": {State: "built", SentAt: &at},
|
||||
"notes": {State: "built"},
|
||||
"later": {},
|
||||
}}
|
||||
rollsOut := func(m string) bool { return m != "notes" }
|
||||
sayUnsent(&p, rollsOut)
|
||||
sayUnsent(&p, rollsOut)
|
||||
if p.Note != "closed by hand at tier 1; built and never sent: agent — `push --behind` sends them" {
|
||||
t.Fatalf("the note reads %q", p.Note)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Whether a mesh could move its bus, read from a real store.
|
||||
//
|
||||
// The readiness reasoning has its own tests; this is about the gathering — that the question is
|
||||
// answered from what the mesh actually holds, on a store with machines and modules in it, without
|
||||
// writing anything.
|
||||
|
||||
func TestReadinessIsGatheredFromWhatTheMeshHolds(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx := context.Background()
|
||||
|
||||
// A mesh mid-change: two machines, the bus module on one of them, a module that speaks and a
|
||||
// module that never does.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "nats", Version: "1", BusUsers: "/var/lib/nats-module/conf/accounts.conf",
|
||||
Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}},
|
||||
{Module: "gitea", Version: "1", Tools: []string{"repo_create"}},
|
||||
{Module: "wallpaper", Version: "1"},
|
||||
} {
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "/r"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.AddNode(ctx, n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "gitea"}, {"laptop", "wallpaper"}} {
|
||||
if _, err := inv.Assign(ctx, a[0], a[1]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
state, err := readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if state.Holder != "anchor" {
|
||||
t.Errorf("the machine holding the bus reads as %q", state.Holder)
|
||||
}
|
||||
if len(state.Nodes) != 2 {
|
||||
t.Errorf("machines read as %v", state.Nodes)
|
||||
}
|
||||
// **A module that never speaks is not counted as missing a credential.** A third of the catalogue
|
||||
// never reaches the bus, and listing those would bury the ones that matter.
|
||||
for _, m := range state.Modules {
|
||||
if strings.HasSuffix(m, "/wallpaper") {
|
||||
t.Errorf("a module that never speaks was counted: %v", state.Modules)
|
||||
}
|
||||
}
|
||||
if len(state.Modules) != 2 {
|
||||
t.Errorf("modules that speak read as %v; expected the bus module and the one with a tool",
|
||||
state.Modules)
|
||||
}
|
||||
|
||||
// Nothing has been minted, so it is not ready — and it says so about each machine by name.
|
||||
why := notReadyOf(state)
|
||||
if len(why) == 0 {
|
||||
t.Fatal("a mesh where nothing has a credential was reported ready to move")
|
||||
}
|
||||
said := strings.Join(why, "\n")
|
||||
for _, name := range []string{"anchor", "laptop"} {
|
||||
if !strings.Contains(said, name) {
|
||||
t.Errorf("the refusal does not name %s: %s", name, said)
|
||||
}
|
||||
}
|
||||
|
||||
// Mint for one machine and it drops out of the complaint, which is how somebody works through it.
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: "node.laptop", Kind: inventory.BusNode, Node: "laptop",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
state, err = readinessOf(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !state.Credentialled["laptop"] {
|
||||
t.Error("a machine that was minted a credential still reads as having none")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A node's own set failing to compose, and the mesh being unable to answer at all, are different
|
||||
// things, and only the first may be passed over when something is gathered across every machine
|
||||
// (novox/hq 04-ISSUES/152). These pin that distinction where the gatherers rely on it.
|
||||
|
||||
func TestASetThatDoesNotComposeIsMarkedAsTheNodesOwnProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{one.Module, two.Module} {
|
||||
if _, err := open.inventory.Assign(t.Context(), "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
_, _, err := planFor(t.Context(), open, "laptop")
|
||||
if err == nil {
|
||||
t.Fatal("two modules claiming one seat composed anyway")
|
||||
}
|
||||
if !unresolvable(err) {
|
||||
t.Fatalf("a set that cannot compose was not marked as the node's own problem: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStoreThatCannotBeReadIsNotANodeThatDoesNotCompose(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
|
||||
// Nothing is wrong with anchor. The question simply cannot be asked.
|
||||
stopped, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
|
||||
_, _, err := planFor(stopped, open, "anchor")
|
||||
if err == nil {
|
||||
t.Fatal("a plan composed against a store that could not be read")
|
||||
}
|
||||
if unresolvable(err) {
|
||||
t.Fatalf("a question the mesh could not answer was read as a node that runs nothing: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// rotateCommand replaces a credential and moves both ends together.
|
||||
//
|
||||
// **This is the invariant novox/hq ADR 0001 records as unowned, and it was measurably false.** On
|
||||
// 2026-08-22 `provision_ensure` — documented as never rotating an existing secret — minted a new
|
||||
// password on every adoption and updated only the provider's row. Consumers on three nodes held
|
||||
// dead credentials for two days; two rows for one provision were written 216 ms apart, so at most
|
||||
// one could match the live role. Nothing enumerated who held the old one, and nothing said so.
|
||||
//
|
||||
// Three things make that impossible here, and all three are deliberate:
|
||||
//
|
||||
// **The holders are a set the mesh can name.** Each pair has its own credential, so rotating one
|
||||
// consumer's password touches one role and leaves every other consumer alone — and the list of who
|
||||
// is affected is a query rather than an assumption.
|
||||
//
|
||||
// **Both ends are pushed by this command, not by a later one.** A rotation that changed the record
|
||||
// and left the sending to whoever remembered is the fault above, exactly.
|
||||
//
|
||||
// **It is all-or-nothing.** If any affected machine cannot be resolved, nothing is sent and the old
|
||||
// credential keeps working — which is a mesh that has not rotated, and is far better than one that
|
||||
// has half-rotated.
|
||||
func rotateCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("rotate", flag.ContinueOnError)
|
||||
// One consumer rather than all of them. Ordinary: a credential is suspected on one machine,
|
||||
// and rotating the other nine would be a great deal of disruption for one suspicion.
|
||||
only := set.String("consumer", "", "only this machine's credential, rather than every holder's")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("rotate <provision> [--consumer <machine>]")
|
||||
}
|
||||
provision := positionals[0]
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
holders, err := inv.HoldersOf(ctx, provision, *only)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(holders) == 0 {
|
||||
// Said, not silent. "Nobody holds this" and "this did not run" must never look the same —
|
||||
// and a rotation somebody believes happened is worse than one they know did not.
|
||||
if *only != "" {
|
||||
return fmt.Errorf(
|
||||
"%s holds no credential for %q, so there is nothing to rotate. `plan %s` says "+
|
||||
"what it does hold", *only, provision, *only)
|
||||
}
|
||||
return fmt.Errorf(
|
||||
"nothing in this mesh holds a credential for %q, so there is nothing to rotate",
|
||||
provision)
|
||||
}
|
||||
|
||||
// Every machine at both ends, named before anything changes. A person about to rotate a
|
||||
// production credential is entitled to know the blast radius before it is the past tense.
|
||||
affected := map[string]bool{}
|
||||
for _, h := range holders {
|
||||
affected[h.Consumer] = true
|
||||
affected[h.Provider] = true
|
||||
}
|
||||
machines := make([]string, 0, len(affected))
|
||||
for name := range affected {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
|
||||
fmt.Printf("rotating %s for %d holder(s):\n", provision, len(holders))
|
||||
for _, h := range holders {
|
||||
// The module, because a machine may hold several credentials for one provision and
|
||||
// rotating "anchor's database password" now means rotating three of them.
|
||||
fmt.Printf(" %s on %s, from %s%s\n", h.ConsumerModule, h.Consumer, h.Provider, asLocal(h.Local))
|
||||
}
|
||||
|
||||
for _, h := range holders {
|
||||
if err := inv.RotateSecret(ctx, h.Provision, h.Consumer, h.ConsumerModule, h.Provider, h.Local); err != nil {
|
||||
// Partly rotated, and said so plainly. What is gone is remade on the next push, so
|
||||
// the remedy is to run this again rather than to repair anything — but a machine
|
||||
// whose secret was discarded and not resent is holding a credential the provider is
|
||||
// about to stop honouring, and that is worth knowing now.
|
||||
return fmt.Errorf(
|
||||
"rotating %s for %s on %s from %s: %w\n\nSome credentials were discarded and "+
|
||||
"not yet sent. Run this again once the cause is fixed",
|
||||
h.Provision, h.ConsumerModule, h.Consumer, h.Provider, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Both ends, in one send.** There is a window either way — a role's password changes on the
|
||||
// provider and the file changes on the consumer, and they cannot be simultaneous — so the
|
||||
// honest thing is to make it as short as the broker allows and to never leave it open across
|
||||
// a command boundary, where it depends on somebody's memory.
|
||||
fmt.Printf("\nsending to both ends:\n")
|
||||
if err := sendTo(ctx, open, machines); err != nil {
|
||||
return fmt.Errorf(
|
||||
"%w\n\nThe old credentials are gone from the mesh and the new ones have not arrived. "+
|
||||
"Nothing on those machines has changed yet, so what is running keeps working "+
|
||||
"until the provider next applies. Fix the cause and run `push --behind`", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\n%d machine(s) told. Until both ends have applied, a consumer whose password "+
|
||||
"changed cannot authenticate — `status` says who is still behind\n", len(machines))
|
||||
return nil
|
||||
}
|
||||
|
||||
// asLocal names the credential inside the consumer where it holds several (ADR 0094).
|
||||
func asLocal(local string) string {
|
||||
if local == "" {
|
||||
return ""
|
||||
}
|
||||
return " (as " + local + ")"
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0207 at the controller's acts: `assign` refuses a module whose resources a
|
||||
// seat nothing on the node holds applies, `unassign` refuses taking the last holder from under its
|
||||
// dependents, and `status` reports what composition does not yet refuse.
|
||||
|
||||
func serviceManagerHolder() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "systemd", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.ServiceManagerSeat, Scope: catalogue.ScopeNode,
|
||||
Serves: []string{"units", "status", "start", "stop", "restart", "enable", "disable", "journal"}}},
|
||||
Resources: []map[string]any{{"id": "systemd", "type": "package", "package": "systemd"}}}
|
||||
}
|
||||
|
||||
func packageManagerHolder() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "pacman", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.PackageManagerSeat, Scope: catalogue.ScopeNode}},
|
||||
Resources: []map[string]any{{"id": "refresh", "type": "service", "unit": "pacman-refresh.timer"}}}
|
||||
}
|
||||
|
||||
func aDaemon() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "sshd", Version: "1",
|
||||
Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}}
|
||||
}
|
||||
|
||||
func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, packageManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
|
||||
_, err := assign(ctx, open, "laptop", "sshd")
|
||||
if err == nil {
|
||||
t.Fatal("sshd went onto a machine nothing holds the service manager of")
|
||||
}
|
||||
for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
assigned, err := open.inventory.Assigned(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if contains(assigned, "sshd") {
|
||||
t.Fatalf("a refused assignment was kept: %v", assigned)
|
||||
}
|
||||
|
||||
// The holders depend on each other, so neither goes on alone — and both go on in one act.
|
||||
if _, err := assign(ctx, open, "laptop", "systemd"); err == nil {
|
||||
t.Fatal("systemd went on alone though its package needs a package manager")
|
||||
}
|
||||
if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil {
|
||||
t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said)
|
||||
}
|
||||
if said, err := assign(ctx, open, "laptop", "sshd"); err != nil {
|
||||
t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) {
|
||||
argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "assign laptop systemd pacman" {
|
||||
t.Errorf("the seat's assign became %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, packageManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err := unassign(ctx, open, "laptop", "systemd")
|
||||
if err == nil {
|
||||
t.Fatal("the service manager came off a machine still running services")
|
||||
}
|
||||
for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not name %q:\n%v", want, err)
|
||||
}
|
||||
}
|
||||
assigned, _ := open.inventory.Assigned(ctx, "laptop")
|
||||
if !contains(assigned, "systemd") {
|
||||
t.Fatalf("a refused unassignment took the module off anyway: %v", assigned)
|
||||
}
|
||||
if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil {
|
||||
t.Fatalf("a dependent could not come off: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) {
|
||||
// The mesh as it ran before the switch (novox/hq ADR 0207 §4).
|
||||
defer catalogue.EnforcingSeatDependencies(false)()
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, serviceManagerHolder())
|
||||
register(t, open, aDaemon())
|
||||
// Assigned straight into the store: a machine whose modules predate the rule, which is every
|
||||
// machine on the day it ships.
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, refused := asked.refused["laptop"]; refused {
|
||||
t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"])
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh with an unheld dependency reads as all well")
|
||||
}
|
||||
got := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("status does not say %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Unheld []catalogue.Unheld `json:"unheld"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat {
|
||||
t.Errorf("the document's unheld is %+v", doc.Unheld)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// What this mesh can have one of, and who fills each (novox/hq ADR 0110).
|
||||
//
|
||||
// **Derived every time, never stored.** A seat is held by a module assignment, so the answer is
|
||||
// computed from assignments by the same resolution that decides what every machine runs. A table
|
||||
// of holders kept beside the assignments would be a second copy of one fact, and the first thing
|
||||
// to be wrong about it.
|
||||
|
||||
// seatHolder is one assignment holding a seat.
|
||||
type seatHolder struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
}
|
||||
|
||||
// seatRow is one seat and who holds it. Unheld is an answer — "this mesh has no X" — not a fault.
|
||||
type seatRow struct {
|
||||
Seat string `json:"seat"`
|
||||
Scope string `json:"scope"`
|
||||
Delivers string `json:"delivers,omitempty"`
|
||||
Decision string `json:"decision"`
|
||||
// Replicated says the seat may be held on several machines at once (novox/hq ADR 0223).
|
||||
Replicated bool `json:"replicated,omitempty"`
|
||||
Holders []seatHolder `json:"holders"`
|
||||
}
|
||||
|
||||
// seatsHeld is every seat the mesh defines with its holders, and every claim held that names no
|
||||
// seat in the set.
|
||||
//
|
||||
// **The second list is not empty by construction.** Manifests are held to the set when they are
|
||||
// registered, and a mesh can hold one registered before the set closed. Leaving its claim out of the
|
||||
// overview would make the one thing the overview is for — what does this mesh have — quietly
|
||||
// incomplete.
|
||||
func seatsHeld(seats []catalogue.Seat, held []catalogue.Held) ([]seatRow, []catalogue.Held) {
|
||||
rows := make([]seatRow, 0, len(seats))
|
||||
for _, s := range seats {
|
||||
row := seatRow{Seat: s.Name, Scope: s.Scope, Delivers: s.Delivers, Decision: s.Decision,
|
||||
Replicated: s.Replicated, Holders: []seatHolder{}}
|
||||
seen := map[seatHolder]bool{}
|
||||
for _, h := range held {
|
||||
// Resolve the held claim to a seat rather than comparing names, so a record naming a
|
||||
// seat's former name groups under it after a rename (novox/hq ADR 0122).
|
||||
hs, ok := catalogue.SeatNamed(h.Claim)
|
||||
if !ok || hs.Name != s.Name || h.Scope != s.Scope {
|
||||
continue
|
||||
}
|
||||
holder := seatHolder{Node: h.Node, Module: h.Module}
|
||||
if !seen[holder] {
|
||||
seen[holder] = true
|
||||
row.Holders = append(row.Holders, holder)
|
||||
}
|
||||
}
|
||||
sort.Slice(row.Holders, func(i, j int) bool {
|
||||
if row.Holders[i].Node != row.Holders[j].Node {
|
||||
return row.Holders[i].Node < row.Holders[j].Node
|
||||
}
|
||||
return row.Holders[i].Module < row.Holders[j].Module
|
||||
})
|
||||
rows = append(rows, row)
|
||||
}
|
||||
var outside []catalogue.Held
|
||||
for _, h := range held {
|
||||
// Outside the set only if it resolves to no seat at all — a former name still resolves.
|
||||
if _, ok := catalogue.SeatNamed(h.Claim); !ok {
|
||||
outside = append(outside, h)
|
||||
}
|
||||
}
|
||||
sort.Slice(outside, func(i, j int) bool {
|
||||
if outside[i].Claim != outside[j].Claim {
|
||||
return outside[i].Claim < outside[j].Claim
|
||||
}
|
||||
return outside[i].Node < outside[j].Node
|
||||
})
|
||||
return rows, outside
|
||||
}
|
||||
|
||||
// seatCommand changes the set — the whole point of it being data (novox/hq ADR 0122) — and, since
|
||||
// ADR 0131, changes who holds a seat.
|
||||
func seatCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 3 && args[0] == "rename" {
|
||||
from, to := args[1], args[2]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if err := open.inventory.RenameSeat(ctx, from, to); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is now %s — its former name still resolves, so nothing is rebuilt, "+
|
||||
"re-registered or frozen (novox/hq ADR 0122)\n", from, to)
|
||||
return nil
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--to" {
|
||||
return handOver(ctx, args[0], args[2], false)
|
||||
}
|
||||
if len(args) == 3 && args[1] == "--add" {
|
||||
return handOver(ctx, args[0], args[2], true)
|
||||
}
|
||||
return fmt.Errorf("seat rename <from> <to> | seat <name> --to <node>/<module> | " +
|
||||
"seat <name> --add <node>/<module>")
|
||||
}
|
||||
|
||||
// handOver makes one assignment the holder of a seat, as one act, so the seat is never without a
|
||||
// holder in between (novox/hq ADR 0131, design 28 task 5.3). The control plane finds its own bus
|
||||
// through one of these seats; the day it was left empty mid-change is why this exists.
|
||||
//
|
||||
// Everything that could make the new holder wrong is refused here, before the row is written: the
|
||||
// seat must exist, the assignment must exist, and the module must be able to hold the seat —
|
||||
// claim it at its scope and provide what it delivers, judged against the store's row. What is
|
||||
// **not** checked is whether the module is running yet: that is what `push` confirms afterwards,
|
||||
// and refusing to record a handover to a module the node has not started would make the handover
|
||||
// impossible to do before the switch instead of as the switch.
|
||||
//
|
||||
// **Or adds one holder beside the others, for a replicated seat** (novox/hq ADR 0223): `--add`
|
||||
// records the named assignment as a further holder and leaves every holder on record as it is. A
|
||||
// seat held once refuses it, naming `--to`; `--to` on a replicated seat replaces every holder with
|
||||
// the one named, as it always did.
|
||||
func handOver(ctx context.Context, seatName, to string, adding bool) error {
|
||||
nodeName, module, ok := strings.Cut(to, "/")
|
||||
if !ok || nodeName == "" || module == "" {
|
||||
return fmt.Errorf("the new holder is named <node>/<module>, not %q", to)
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
seat, known := catalogue.SeatNamed(seatName)
|
||||
if !known {
|
||||
return fmt.Errorf("%q is not a seat this mesh defines — `seats` lists them", seatName)
|
||||
}
|
||||
if adding && !seat.Replicated {
|
||||
return fmt.Errorf("%s is held once per %s, so a second holder cannot be added beside the first — "+
|
||||
"`seat %s --to %s` hands it over", seat.Name, seat.Scope, seat.Name, to)
|
||||
}
|
||||
assigned, err := inv.Assigned(ctx, nodeName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.Contains(assigned, module) {
|
||||
return fmt.Errorf("%s is not assigned to %s, so it cannot hold anything there — "+
|
||||
"`assign %s %s` first", module, nodeName, nodeName, module)
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m *catalogue.Manifest
|
||||
for i := range entries {
|
||||
if entries[i].Manifest.Module == module {
|
||||
m = &entries[i].Manifest
|
||||
}
|
||||
}
|
||||
if m == nil {
|
||||
return fmt.Errorf("%s is assigned but not in the catalogue, which should not happen", module)
|
||||
}
|
||||
var was string
|
||||
var held []string
|
||||
holdings, err := inv.Holdings(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, h := range holdings {
|
||||
if hs, ok := catalogue.SeatNamed(h.Claim); ok && hs.Name == seat.Name {
|
||||
was = h.Node
|
||||
held = append(held, h.Node)
|
||||
}
|
||||
}
|
||||
|
||||
// **Recording who already holds the seat is not making a new holder, and is not judged like
|
||||
// one.** On a mesh that predates the record, the first handover has to begin by writing down
|
||||
// the standing holder — otherwise the next holder cannot be assigned beside it, because two
|
||||
// eligible claimants with nothing on record are refused. That standing holder may no longer
|
||||
// satisfy what the seat delivers (the row moved under it, on purpose, as ADR 0131's first step),
|
||||
// and it holds regardless: derivation never read that column. So when nothing is on record and
|
||||
// the named assignment is the one holding by derivation, only the claim itself is checked here.
|
||||
// Every *change* of holder is judged in full.
|
||||
claimsIt := false
|
||||
for _, c := range m.Claims {
|
||||
if cs, ok := catalogue.SeatNamed(c.Name); ok && cs.Name == seat.Name && c.At() == seat.Scope {
|
||||
claimsIt = true
|
||||
}
|
||||
}
|
||||
if was == "" && claimsIt {
|
||||
fmt.Printf("nothing was on record for %s; recording %s on %s as its standing holder\n",
|
||||
seat.Name, module, nodeName)
|
||||
} else if err := catalogue.CanHold(*m, seat); err != nil {
|
||||
return fmt.Errorf("%s cannot hold %s: %w", module, seat.Name, err)
|
||||
}
|
||||
if adding {
|
||||
if err := inv.AddSeatHolder(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s, beside what was on record: %s\n", seat.Name, module, nodeName,
|
||||
strings.Join(held, ", "))
|
||||
fmt.Printf(" `push --behind` re-declares every machine that reads the seat's holders\n")
|
||||
return nil
|
||||
}
|
||||
if err := inv.HoldSeat(ctx, seat.Name, seat.Scope, nodeName, module); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held by %s on %s\n", seat.Name, module, nodeName)
|
||||
if was != "" && was != nodeName {
|
||||
fmt.Printf(" `push %s` and `push %s` send both machines what changed\n", was, nodeName)
|
||||
} else {
|
||||
fmt.Printf(" `push %s` sends the machine what changed; every other machine that reads the "+
|
||||
"seat is re-declared by `push --behind`\n", nodeName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func seatsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("seats", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the same, as JSON")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Every node, none excluded: the same view of what each machine holds that planning uses.
|
||||
world, err := theRestOfTheMesh(ctx, inv, shelf, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, outside := seatsHeld(catalogue.Seats(), world.Held)
|
||||
|
||||
if *asJSON {
|
||||
out := struct {
|
||||
Seats []seatRow `json:"seats"`
|
||||
Outside []catalogue.Held `json:"outside,omitempty"`
|
||||
}{rows, outside}
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(w, "SEAT\tSCOPE\tDELIVERS\tHELD BY")
|
||||
for _, r := range rows {
|
||||
delivers := r.Delivers
|
||||
if delivers == "" {
|
||||
delivers = "—"
|
||||
}
|
||||
holders := "unheld"
|
||||
if len(r.Holders) > 0 {
|
||||
parts := make([]string, 0, len(r.Holders))
|
||||
for _, h := range r.Holders {
|
||||
parts = append(parts, h.Module+" on "+h.Node)
|
||||
}
|
||||
holders = strings.Join(parts, ", ")
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Seat, r.Scope, delivers, holders)
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(outside) > 0 {
|
||||
fmt.Println("\nheld, and not a seat this mesh defines (registered before the set closed — novox/hq ADR 0110):")
|
||||
for _, h := range outside {
|
||||
fmt.Printf(" %s %s on %s\n", h.Claim, h.Module, h.Node)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The overview of what a mesh has (novox/hq ADR 0110).
|
||||
|
||||
func TestEverySeatIsListedIncludingTheOnesNobodyHolds(t *testing.T) {
|
||||
// An unheld seat is an answer — "this mesh has no forge" — so it is listed rather than omitted.
|
||||
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "postgres"},
|
||||
})
|
||||
if len(rows) != len(catalogue.Seats()) {
|
||||
t.Fatalf("%d seats listed of %d", len(rows), len(catalogue.Seats()))
|
||||
}
|
||||
for _, r := range rows {
|
||||
switch r.Seat {
|
||||
case "mesh-store":
|
||||
if len(r.Holders) != 1 || r.Holders[0].Module != "postgres" || r.Holders[0].Node != "anchor" {
|
||||
t.Errorf("mesh-store is held by %+v", r.Holders)
|
||||
}
|
||||
if r.Delivers != "postgres-database" {
|
||||
t.Errorf("mesh-store does not say what it delivers: %q", r.Delivers)
|
||||
}
|
||||
case "git":
|
||||
if len(r.Holders) != 0 {
|
||||
t.Errorf("git is held by %+v in a mesh with no forge", r.Holders)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeSeatListsEveryMachineHoldingIt(t *testing.T) {
|
||||
rows, _ := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "node2", Module: "nftables"},
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||
// Resolved twice, reported once: a machine is one holder however many passes saw it.
|
||||
{Claim: "node-packet-filter", Scope: catalogue.ScopeNode, Node: "anchor", Module: "nftables"},
|
||||
})
|
||||
for _, r := range rows {
|
||||
if r.Seat != "node-packet-filter" {
|
||||
continue
|
||||
}
|
||||
if len(r.Holders) != 2 || r.Holders[0].Node != "anchor" || r.Holders[1].Node != "node2" {
|
||||
t.Fatalf("the packet filter is held by %+v", r.Holders)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatal("the packet filter is not listed")
|
||||
}
|
||||
|
||||
func TestAClaimOutsideTheSetIsShownNotHidden(t *testing.T) {
|
||||
// A manifest registered before the set closed can still hold one. Leaving it out would make
|
||||
// the overview quietly incomplete, which is the one thing it may not be.
|
||||
_, outside := seatsHeld(catalogue.Seats(), []catalogue.Held{
|
||||
{Claim: "the-controller", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "mesh-controller"},
|
||||
})
|
||||
if len(outside) != 1 || outside[0].Claim != "the-controller" {
|
||||
t.Fatalf("a claim outside the set was not shown: %+v", outside)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/nats-io/nats.go/micro"
|
||||
"os"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The mesh's own verbs, served as the mesh-controller seat's tools (novox/hq ADR 0154, design 33).
|
||||
//
|
||||
// **Each tool runs the command it names, in this same binary, and answers what it printed.** That is
|
||||
// ADR 0035 taken literally: the logic lives once, in the command, and a surface is an adapter with no
|
||||
// decisions in it. Running a fresh process rather than calling the function keeps two things true
|
||||
// that calling it would not — every command opens and closes its own stores the way it does from a
|
||||
// shell, and nothing a command prints to the process's standard output can leak into another call's
|
||||
// answer. It also means a refusal is the same refusal in the same words, because it is the same
|
||||
// output.
|
||||
|
||||
// verbAnswer is what a verb answers: what the command printed, whether it succeeded, and — where the
|
||||
// command speaks JSON — the same as data.
|
||||
type verbAnswer struct {
|
||||
Output string `json:"output"`
|
||||
OK bool `json:"ok"`
|
||||
Answer any `json:"answer,omitempty"`
|
||||
}
|
||||
|
||||
// argvFor is the command line a verb and its arguments become. Only the verbs the seat declares, and
|
||||
// only the arguments each declares: a caller cannot reach a flag the schema did not name.
|
||||
//
|
||||
// **Nothing a caller sends is passed over** (novox/hq issue 244). An argument the verb does not
|
||||
// declare is refused, naming it; a switch that is not "true" or "false" is refused; and an argument
|
||||
// the verb declares but did not use for the command line it composed — given beside another that
|
||||
// wins, or half of a shape — is refused too. On 2026-10-05 a push naming one machine reached the
|
||||
// verb without the machine and ran as a push of every machine behind; a verb that answers "I did
|
||||
// not take that" would have stopped it before anything was sent.
|
||||
func argvFor(verb string, args map[string]any) ([]string, error) {
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv, err := a.commandLine()
|
||||
if len(a.misread) > 0 {
|
||||
// The table and the command line disagree: the verb reads an argument no caller can see
|
||||
// in its schema, so no caller could ever pass it.
|
||||
return nil, fmt.Errorf("%s reads %s, which its schema does not declare — this build's verb "+
|
||||
"table and its command lines disagree", verb, quoteAll(a.misread))
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if unused := a.unused(); len(unused) > 0 {
|
||||
return nil, fmt.Errorf("%s did not use %s together with %s, and an argument a verb would pass over "+
|
||||
"is refused: nothing was done", verb, quoteAll(unused), quoteAll(a.usedGiven()))
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
|
||||
// verbArguments are one call's arguments, checked against the verb's schema, and which of them the
|
||||
// command line was composed from.
|
||||
type verbArguments struct {
|
||||
verb string
|
||||
given map[string]string
|
||||
used map[string]bool
|
||||
declared map[string]bool
|
||||
misread []string // arguments the command line read that the schema does not declare: a bug here
|
||||
}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
// wrote.
|
||||
func controllerVerb(name string) (catalogue.Verb, bool) {
|
||||
for _, v := range catalogue.ControllerVerbs {
|
||||
if v.Name == name {
|
||||
return v, true
|
||||
}
|
||||
}
|
||||
return catalogue.Verb{}, false
|
||||
}
|
||||
|
||||
// declaredArguments are a schema's properties, and which of them are switches.
|
||||
func declaredArguments(v catalogue.Verb) (names []string, switches map[string]bool) {
|
||||
switches = map[string]bool{}
|
||||
props, _ := v.Input["properties"].(map[string]any)
|
||||
for name, p := range props {
|
||||
names = append(names, name)
|
||||
desc, _ := p.(map[string]any)
|
||||
switch enum := desc["enum"].(type) {
|
||||
case []string:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
case []any:
|
||||
switches[name] = len(enum) == 2 && enum[0] == "true" && enum[1] == "false"
|
||||
}
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names, switches
|
||||
}
|
||||
|
||||
// readArguments refuses what the verb does not take, before anything is composed.
|
||||
func readArguments(verb string, args map[string]any) (*verbArguments, error) {
|
||||
v, known := controllerVerb(verb)
|
||||
if !known {
|
||||
return nil, fmt.Errorf("%q is not a verb the %s seat serves", verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
declared := map[string]bool{}
|
||||
for _, n := range names {
|
||||
declared[n] = true
|
||||
}
|
||||
takes := "none"
|
||||
if len(names) > 0 {
|
||||
takes = quoteAll(names)
|
||||
}
|
||||
a := &verbArguments{verb: verb, given: map[string]string{}, used: map[string]bool{}, declared: declared}
|
||||
keys := make([]string, 0, len(args))
|
||||
for k := range args {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
if !declared[k] {
|
||||
return nil, fmt.Errorf("%s takes no argument %q — it takes %s; nothing was done", verb, k, takes)
|
||||
}
|
||||
var value string
|
||||
switch x := args[k].(type) {
|
||||
case nil:
|
||||
continue
|
||||
case string:
|
||||
value = strings.TrimSpace(x)
|
||||
case bool:
|
||||
if !switches[k] {
|
||||
return nil, fmt.Errorf("%s: %q is text, not true or false", verb, k)
|
||||
}
|
||||
value = fmt.Sprint(x)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is text, and was given %T", verb, k, x)
|
||||
}
|
||||
if switches[k] {
|
||||
switch value {
|
||||
case "true":
|
||||
case "false", "":
|
||||
continue // said and off: the same as not given, and nothing passed over
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: %q is \"true\" or \"false\", not %q", verb, k, value)
|
||||
}
|
||||
}
|
||||
if value != "" {
|
||||
a.given[k] = value
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// str is one argument's value, marked as used.
|
||||
func (a *verbArguments) str(key string) string {
|
||||
if !a.declared[key] {
|
||||
a.misread = append(a.misread, key)
|
||||
}
|
||||
a.used[key] = true
|
||||
return a.given[key]
|
||||
}
|
||||
|
||||
// on is a switch, marked as used.
|
||||
func (a *verbArguments) on(key string) bool { return a.str(key) == "true" }
|
||||
|
||||
// need refuses a call missing a required argument, in the verb's own words.
|
||||
func (a *verbArguments) need(keys ...string) error {
|
||||
for _, k := range keys {
|
||||
if a.str(k) == "" {
|
||||
return fmt.Errorf("%s needs %q", a.verb, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unused are the arguments given that the command line was not composed from.
|
||||
func (a *verbArguments) unused() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if !a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *verbArguments) usedGiven() []string {
|
||||
var out []string
|
||||
for k := range a.given {
|
||||
if a.used[k] {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
return []string{"nothing"}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func quoteAll(xs []string) string {
|
||||
q := make([]string, len(xs))
|
||||
for i, x := range xs {
|
||||
if x == "nothing" {
|
||||
q[i] = x
|
||||
continue
|
||||
}
|
||||
q[i] = fmt.Sprintf("%q", x)
|
||||
}
|
||||
return strings.Join(q, ", ")
|
||||
}
|
||||
|
||||
// commandLine composes the command. Every argument it reads is one it uses: a branch that reads an
|
||||
// argument and then drops it would pass it over, which is what the check after it exists to refuse.
|
||||
func (a *verbArguments) commandLine() ([]string, error) {
|
||||
verb, str, on, need := a.verb, a.str, a.on, a.need
|
||||
switch verb {
|
||||
case "command":
|
||||
// The generic verb: the command line as given, split as a shell would split it, with
|
||||
// nothing added — the named verbs add flags a caller cannot reach; this one is the whole
|
||||
// binary and says so in its description (novox/hq ADR 0154, 0175).
|
||||
if err := need("command"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv, err := splitCommandLine(str("command"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(argv) == 0 {
|
||||
return nil, errors.New("command names no command")
|
||||
}
|
||||
return argv, nil
|
||||
case "tools":
|
||||
return nil, errors.New("tools is answered from the records, not by a command")
|
||||
case "status":
|
||||
return []string{"status", "--json"}, nil
|
||||
case "nodes":
|
||||
return []string{"node", "list", "--json"}, nil
|
||||
case "node":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"node", "show", str("node")}, nil
|
||||
case "modules":
|
||||
return []string{"module", "list", "--json"}, nil
|
||||
case "seats":
|
||||
return []string{"seats", "--json"}, nil
|
||||
case "builds":
|
||||
if id := str("log"); id != "" {
|
||||
return []string{"builds", "--log", id}, nil
|
||||
}
|
||||
argv := []string{"builds"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
if m := str("module"); m != "" {
|
||||
argv = append(argv, m)
|
||||
}
|
||||
return argv, nil
|
||||
case "plans":
|
||||
if r := str("repository"); r != "" {
|
||||
argv := []string{"plans", "--what-if", r}
|
||||
if p := str("paths"); p != "" {
|
||||
argv = append(argv, "--paths", p)
|
||||
}
|
||||
if m := str("modules"); m != "" {
|
||||
argv = append(argv, "--modules", m)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
for _, act := range []string{"stop", "close", "retry"} {
|
||||
if id := str(act); id != "" {
|
||||
return []string{"plans", act, id}, nil
|
||||
}
|
||||
}
|
||||
if id := str("id"); id != "" {
|
||||
return []string{"plans", id}, nil
|
||||
}
|
||||
argv := []string{"plans"}
|
||||
if n := str("limit"); n != "" {
|
||||
argv = append(argv, "-n", n)
|
||||
}
|
||||
return argv, nil
|
||||
// The build queue (novox/hq ADR 0219).
|
||||
case "queue":
|
||||
return []string{"queue"}, nil
|
||||
case "cancel", "kill":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{verb, str("id")}, nil
|
||||
case "clear":
|
||||
if on("dead") {
|
||||
return []string{"clear", "--dead"}, nil
|
||||
}
|
||||
return []string{"clear"}, nil
|
||||
case "rebuild":
|
||||
if err := need("what"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"rebuild", str("what")}, nil
|
||||
case "replay":
|
||||
if err := need("id"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"replay", str("id")}
|
||||
if on("register") {
|
||||
argv = append(argv, "--register")
|
||||
}
|
||||
if on("older") {
|
||||
argv = append(argv, "--older")
|
||||
}
|
||||
return argv, nil
|
||||
case "pause", "resume":
|
||||
if n := str("node"); n != "" {
|
||||
return []string{verb, n}, nil
|
||||
}
|
||||
return []string{verb}, nil
|
||||
case "plan":
|
||||
if err := need("node"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if on("files") {
|
||||
return []string{"plan", str("node"), "--files"}, nil
|
||||
}
|
||||
return []string{"plan", str("node"), "--json"}, nil
|
||||
case "assign", "unassign":
|
||||
if err := need("node", "module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Several modules comma-separated, judged as one act (novox/hq ADR 0207): the holders of
|
||||
// the seats that apply resources depend on each other and go on together.
|
||||
return append([]string{verb, str("node")}, splitModules(str("module"))...), nil
|
||||
case "pin":
|
||||
if err := need("node", "provision", "from", "module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"pin", str("node"), str("provision"), str("from"), str("module")}, nil
|
||||
case "unpin":
|
||||
if err := need("node", "provision"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"unpin", str("node"), str("provision")}, nil
|
||||
case "push":
|
||||
// Sent and not waited for: the asker reads `status` for what the machine did, which is
|
||||
// what a person at a shell does too. A tool call that blocked for a push's whole apply would
|
||||
// time out on every machine that takes a minute, and say nothing about the ones that did not.
|
||||
if n := str("node"); n != "" {
|
||||
// behind is not read here: given with a machine, it is refused as passed over — naming
|
||||
// a machine and asking for every machine behind are two requests, and guessing one
|
||||
// would push a machine nobody named, or not push one somebody did.
|
||||
return []string{"push", n, "--wait", "0"}, nil
|
||||
}
|
||||
// No machine: the whole mesh, whether or not behind said so. The command's answer says it
|
||||
// first, so a caller who meant one machine reads that it was not one.
|
||||
on("behind")
|
||||
return []string{"push", "--behind", "--wait", "0"}, nil
|
||||
case "rotate":
|
||||
if p := str("provision"); p != "" {
|
||||
argv := []string{"rotate", p}
|
||||
if c := str("consumer"); c != "" {
|
||||
argv = append(argv, "--consumer", c)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
_, node := a.given["node"]
|
||||
_, module := a.given["module"]
|
||||
_, secret := a.given["secret"]
|
||||
if node || module || secret {
|
||||
if err := need("node", "module", "secret"); err != nil {
|
||||
return nil, fmt.Errorf("%w: a module's own secret is named by node, module and secret together", err)
|
||||
}
|
||||
return []string{"secret", "rotate", str("node"), str("module"), str("secret")}, nil
|
||||
}
|
||||
// Neither shape: the command says its usage, which names both, and that is the answer the
|
||||
// caller needs.
|
||||
return []string{"rotate"}, nil
|
||||
case "settings":
|
||||
// `settings set|clear` at a shell (novox/hq issue 198). The values travel as an argument
|
||||
// because a tool has no file to hand the command; the command reads either.
|
||||
if err := need("module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var argv []string
|
||||
if on("clear") {
|
||||
argv = []string{"settings", "clear", str("module")}
|
||||
} else {
|
||||
argv = []string{"settings", "set", str("module")}
|
||||
// Neither values nor clear: the command says its usage, which names both.
|
||||
if v := str("values"); v != "" {
|
||||
argv = append(argv, v)
|
||||
}
|
||||
}
|
||||
if n := str("node"); n != "" {
|
||||
argv = append(argv, "--node", n)
|
||||
}
|
||||
return argv, nil
|
||||
case "issue":
|
||||
// The same act as `module issue` at a shell (novox/hq design 25 §4): the account is minted
|
||||
// into the mesh's records and delivered at the machine's next push, which is the caller's to
|
||||
// ask for — so the mesh is never pushed as a side effect of a credential.
|
||||
if err := need("node", "module"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"module", "issue", str("module"), "--node", str("node")}, nil
|
||||
case "build":
|
||||
if err := need("repository"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Not waited for: a tool call cannot hold a connection for the minutes a build takes; the
|
||||
// daemon takes the outcome in when it comes and the id follows the build (issue 176). A
|
||||
// repository given without a scheme is a path on the forge holding the git seat.
|
||||
argv := []string{"build", str("repository"), "--wait", "0"}
|
||||
if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") {
|
||||
argv = append(argv, "--self")
|
||||
}
|
||||
if p := str("path"); p != "" {
|
||||
argv = append(argv, "--path", p)
|
||||
}
|
||||
if r := str("ref"); r != "" {
|
||||
argv = append(argv, "--ref", r)
|
||||
}
|
||||
return argv, nil
|
||||
}
|
||||
return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for",
|
||||
verb, catalogue.ControllerSeatName)
|
||||
}
|
||||
|
||||
// jsonVerbs are the verbs whose command speaks JSON, so the answer carries it as data as well.
|
||||
var jsonVerbs = map[string]bool{"status": true, "seats": true, "plan": true, "collection": true}
|
||||
|
||||
// runVerb runs this binary with the given command line and gathers what it said.
|
||||
func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return verbAnswer{}, err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, self, argv...)
|
||||
// The same environment: the stores' credentials, the bus, the broker — everything a command run
|
||||
// from a shell in this container would have, because it is that.
|
||||
cmd.Env = os.Environ()
|
||||
// Two buffers, one answer. What the command *says* is both streams, in the order a person at
|
||||
// a shell would read them; what it *answers as data* is standard output alone — `status --json`
|
||||
// prints its warnings beside the document, and a JSON parsed from the two together parsed
|
||||
// nothing (2026-09-30, the first status asked through the console had no `answer`).
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
runErr := cmd.Run()
|
||||
answer := verbAnswer{Output: stdout.String() + stderr.String(), OK: runErr == nil}
|
||||
if jsonVerbs[argv[0]] && runErr == nil {
|
||||
var parsed any
|
||||
if json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &parsed) == nil {
|
||||
answer.Answer = parsed
|
||||
}
|
||||
}
|
||||
var exit *exec.ExitError
|
||||
if runErr != nil && !errors.As(runErr, &exit) {
|
||||
// Not the command refusing — the command not running at all, which is this process's fault.
|
||||
return answer, fmt.Errorf("could not run %s: %w", strings.Join(argv, " "), runErr)
|
||||
}
|
||||
return answer, nil
|
||||
}
|
||||
|
||||
// seatToolHandlers are the handlers for every verb the mesh-controller seat declares, from the
|
||||
// store's row, so a verb the row does not carry is not served. A verb it carries that this binary
|
||||
// cannot run is named at start and answers the reason when called — never a refusal to serve, which
|
||||
// would take the whole control plane down for one word (novox/hq ADR 0185).
|
||||
func seatToolHandlers() (map[string]link.ToolHandler, []string, error) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
return nil, nil, fmt.Errorf("this mesh defines no %s seat", catalogue.ControllerSeatName)
|
||||
}
|
||||
var behind []string
|
||||
handlers := map[string]link.ToolHandler{}
|
||||
for _, v := range seat.Serves {
|
||||
verb := v.Name
|
||||
if inProcess[verb] {
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
if len(bytes.TrimSpace(raw)) > 0 {
|
||||
if err := json.Unmarshal(raw, &args); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||
}
|
||||
}
|
||||
// Refused like any verb's: what a verb does not take is not ignored.
|
||||
a, err := readArguments(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if verb == "calls" {
|
||||
return callsAnswer(link.Calls, a.given["call"])
|
||||
}
|
||||
return seatTools(), nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := argvFor(verb, sampleArguments(v)); err != nil {
|
||||
// **A row ahead of this binary is not a reason to go silent.**
|
||||
//
|
||||
// The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb
|
||||
// this build does not know means the row was widened by a newer one — the ordinary
|
||||
// state of a roll-out, and of a push that put an older control plane back. Refusing to
|
||||
// serve at all made that transient fatal: on 2026-10-02 one unknown verb took the whole
|
||||
// mesh off the bus for ten minutes, and the way back was a human running the binary by
|
||||
// hand, because the thing that would have repaired it is the thing that was down
|
||||
// (novox/hq 04-ISSUES/201, ADR 0185).
|
||||
//
|
||||
// So the verbs this binary knows are served, and this one answers the reason instead of
|
||||
// nothing: a caller gets a sentence naming the fault, and everything else keeps working
|
||||
// — including the push that replaces this binary with the one whose verb it is.
|
||||
behind = append(behind, verb)
|
||||
reason := err
|
||||
handlers[verb] = func(context.Context, json.RawMessage) (any, error) {
|
||||
return nil, fmt.Errorf("%s is in this mesh's %s row and the control plane running "+
|
||||
"here cannot run it: %w. It is a verb of a newer build; this one is behind",
|
||||
verb, catalogue.ControllerSeatName, reason)
|
||||
}
|
||||
continue
|
||||
}
|
||||
handlers[verb] = func(ctx context.Context, raw json.RawMessage) (any, error) {
|
||||
args := map[string]any{}
|
||||
if len(raw) > 0 {
|
||||
if err := json.Unmarshal(raw, &args); err != nil {
|
||||
return nil, fmt.Errorf("the arguments are not a JSON object: %w", err)
|
||||
}
|
||||
}
|
||||
argv, err := argvFor(verb, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if answersFirst(argv) {
|
||||
// Before anything is sent: a push sends the bus's own machine first, and a broker
|
||||
// reloading its user list forgets the answer it was about to permit (novox/hq issue 265).
|
||||
link.Acknowledge(ctx)
|
||||
}
|
||||
return runVerb(ctx, argv)
|
||||
}
|
||||
}
|
||||
return handlers, behind, nil
|
||||
}
|
||||
|
||||
// inProcess are the verbs answered by this process rather than by a command it runs: `tools` from
|
||||
// the records, `calls` from what this process served.
|
||||
var inProcess = map[string]bool{"tools": true, "calls": true}
|
||||
|
||||
// answersFirst is a command line whose caller is answered before it runs: a push, by its verb or
|
||||
// through `command`. A push sends the machine holding the bus first when its user list changed, the
|
||||
// broker reloads, and a reload forgets every answer the bus was about to permit — so an answer
|
||||
// waiting for the push to end was refused, every time the list had changed (novox/hq issue 265).
|
||||
func answersFirst(argv []string) bool {
|
||||
return len(argv) > 0 && argv[0] == "push"
|
||||
}
|
||||
|
||||
// callsAnswer is what `calls` answers: the kept calls, newest first, without their answers — or
|
||||
// one call whole.
|
||||
func callsAnswer(log *link.CallLog, id string) (any, error) {
|
||||
if id != "" {
|
||||
c, ok := log.Get(id)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no call %s is kept here: calls are kept by the controller that "+
|
||||
"answered them, the last %d, and not across a restart — `calls` lists them", id, link.KeptCalls)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
recent := log.Recent()
|
||||
for i := range recent {
|
||||
recent[i].Answer = nil
|
||||
}
|
||||
return map[string]any{"calls": recent, "kept": link.KeptCalls,
|
||||
"note": "newest first; `calls` with a call's id gives its whole answer"}, nil
|
||||
}
|
||||
|
||||
// seatTools is what `tools` answers: every seat with a protocol, and the tools each serves, from the
|
||||
// mesh's own records — no holder in the path, so it is true while a holder restarts (design 33 §5).
|
||||
func seatTools() map[string]any {
|
||||
var seats []map[string]any
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if len(s.Serves) == 0 {
|
||||
continue
|
||||
}
|
||||
var tools []map[string]any
|
||||
for _, v := range s.Serves {
|
||||
tools = append(tools, map[string]any{
|
||||
"name": v.Name, "description": v.Description, "input": v.Input, "output": v.Output,
|
||||
})
|
||||
}
|
||||
seats = append(seats, map[string]any{"seat": s.Name, "scope": s.Scope, "tools": tools})
|
||||
}
|
||||
return map[string]any{"seats": seats}
|
||||
}
|
||||
|
||||
// sampleArguments is one of every argument a verb's schema requires, so the check at start proves the
|
||||
// verb runnable rather than that it happens to want the arguments the check guessed — and nothing
|
||||
// more, since an argument a verb does not declare is refused.
|
||||
func sampleArguments(v catalogue.Verb) map[string]any {
|
||||
sample := map[string]any{}
|
||||
switch required := v.Input["required"].(type) {
|
||||
case []string:
|
||||
for _, k := range required {
|
||||
sample[k] = "x"
|
||||
}
|
||||
case []any:
|
||||
for _, k := range required {
|
||||
if name, ok := k.(string); ok {
|
||||
sample[name] = "x"
|
||||
}
|
||||
}
|
||||
}
|
||||
return sample
|
||||
}
|
||||
|
||||
// splitCommandLine splits a command line into words the way a POSIX shell does for the simple
|
||||
// cases a controller command needs: spaces separate, single or double quotes group, a backslash
|
||||
// escapes the next character inside double quotes or outside any. No expansion of anything.
|
||||
func splitCommandLine(line string) ([]string, error) {
|
||||
var words []string
|
||||
var cur strings.Builder
|
||||
inWord := false
|
||||
quote := rune(0)
|
||||
runes := []rune(line)
|
||||
for i := 0; i < len(runes); i++ {
|
||||
r := runes[i]
|
||||
switch {
|
||||
case quote == '\'':
|
||||
if r == '\'' {
|
||||
quote = 0
|
||||
} else {
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
case quote == '"':
|
||||
if r == '"' {
|
||||
quote = 0
|
||||
} else if r == '\\' && i+1 < len(runes) {
|
||||
i++
|
||||
cur.WriteRune(runes[i])
|
||||
} else {
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
case r == '\'' || r == '"':
|
||||
quote = r
|
||||
inWord = true
|
||||
case r == '\\' && i+1 < len(runes):
|
||||
i++
|
||||
cur.WriteRune(runes[i])
|
||||
inWord = true
|
||||
case r == ' ' || r == '\t' || r == '\n':
|
||||
if inWord {
|
||||
words = append(words, cur.String())
|
||||
cur.Reset()
|
||||
inWord = false
|
||||
}
|
||||
default:
|
||||
cur.WriteRune(r)
|
||||
inWord = true
|
||||
}
|
||||
}
|
||||
if quote != 0 {
|
||||
return nil, fmt.Errorf("command has an unclosed %c quote", quote)
|
||||
}
|
||||
if inWord {
|
||||
words = append(words, cur.String())
|
||||
}
|
||||
return words, nil
|
||||
}
|
||||
|
||||
// seatAnnouncement is what the controller says it serves on the bus (novox/hq ADR 0197): the
|
||||
// mesh-controller seat, one endpoint per verb it answers, each with the seat's own description and
|
||||
// argument schema — the same facts `tools` answers from the records, as NATS's services format.
|
||||
func seatAnnouncement(handlers map[string]link.ToolHandler) micro.Info {
|
||||
about := map[string]catalogue.Verb{}
|
||||
for _, s := range catalogue.SeatsWithAProtocol() {
|
||||
if s.Name == catalogue.ControllerSeatName {
|
||||
for _, v := range s.Serves {
|
||||
about[v.Name] = v
|
||||
}
|
||||
}
|
||||
}
|
||||
verbs := make([]string, 0, len(handlers))
|
||||
for verb := range handlers {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
var endpoints []micro.EndpointInfo
|
||||
for _, verb := range verbs {
|
||||
schema, _ := json.Marshal(about[verb].Input)
|
||||
// The same shape every tool runtime announces in (node-tools' announce package): the name is
|
||||
// `<seat>__<verb>`, as the protocol's characters allow; the metadata is what identifies it.
|
||||
endpoints = append(endpoints, micro.EndpointInfo{
|
||||
Name: catalogue.ControllerSeatName + "__" + verb,
|
||||
Subject: link.SeatToolSubject(catalogue.ControllerSeatName, verb),
|
||||
QueueGroup: "seat." + catalogue.ControllerSeatName,
|
||||
Metadata: map[string]string{
|
||||
"kind": "seat", "module": catalogue.ControllerSeatName, "tool": verb,
|
||||
"seat": catalogue.ControllerSeatName, "scope": "mesh", "interchangeable": "false",
|
||||
"description": about[verb].Description, "schema": string(schema),
|
||||
},
|
||||
})
|
||||
}
|
||||
return micro.Info{
|
||||
ServiceIdentity: micro.ServiceIdentity{
|
||||
Name: catalogue.ControllerSeatName, ID: "controller", Version: "0.1.0",
|
||||
Metadata: map[string]string{"seat": catalogue.ControllerSeatName, "scope": "mesh"},
|
||||
},
|
||||
Description: "the mesh's own verbs, answered by the holder of the mesh-controller seat",
|
||||
Endpoints: endpoints,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,365 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// The schemas the controller serves, verb by verb, as the console receives them: from the
|
||||
// announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244).
|
||||
func servedSchemas(t *testing.T) map[string]catalogue.Verb {
|
||||
t.Helper()
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
served := map[string]catalogue.Verb{}
|
||||
for _, e := range seatAnnouncement(handlers).Endpoints {
|
||||
var input map[string]any
|
||||
if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil {
|
||||
t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err)
|
||||
}
|
||||
served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input}
|
||||
}
|
||||
if len(served) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// subsetsOf is every subset of the names, the empty one included.
|
||||
func subsetsOf(names []string) [][]string {
|
||||
var out [][]string
|
||||
for mask := 0; mask < 1<<len(names); mask++ {
|
||||
var s []string
|
||||
for i, n := range names {
|
||||
if mask&(1<<i) != 0 {
|
||||
s = append(s, n)
|
||||
}
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func argumentsFor(subset []string, switches map[string]bool) map[string]any {
|
||||
args := map[string]any{}
|
||||
for _, n := range subset {
|
||||
if switches[n] {
|
||||
args[n] = "true"
|
||||
} else {
|
||||
args[n] = "x-" + n
|
||||
}
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// saidOutright are the switches that say the default aloud, so the line is the same without them —
|
||||
// on purpose, and only these.
|
||||
var saidOutright = map[string]string{
|
||||
"push": "behind", // the whole mesh is what no machine means; behind lets a caller say they meant it
|
||||
}
|
||||
|
||||
// **No argument a caller gives is passed over** (novox/hq issue 244). For every verb served and
|
||||
// every combination of the arguments its schema declares, the verb either refuses the call, or
|
||||
// composes a command line that each given argument changed: taking any one away changes the line
|
||||
// or makes it refused. An argument the line is the same without is one the verb ignored — the
|
||||
// shape of the push that named a machine and pushed every machine behind.
|
||||
func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
if inProcess[name] {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
args := argumentsFor(subset, switches)
|
||||
argv, err := argvFor(name, args)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "does not declare") {
|
||||
t.Errorf("%s %v: %v", name, args, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, dropped := range subset {
|
||||
fewer := map[string]any{}
|
||||
for k, val := range args {
|
||||
if k != dropped {
|
||||
fewer[k] = val
|
||||
}
|
||||
}
|
||||
without, err := argvFor(name, fewer)
|
||||
if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped {
|
||||
t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with
|
||||
// what it requires and one argument more; and `node` in particular, for every verb whose schema
|
||||
// does not take a machine.
|
||||
func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
strangers := []string{"no-such-argument"}
|
||||
if !contains(names, "node") {
|
||||
strangers = append(strangers, "node")
|
||||
}
|
||||
for _, stranger := range strangers {
|
||||
args := sampleArguments(v)
|
||||
args[stranger] = "x"
|
||||
_, err := argvFor(name, args)
|
||||
if inProcess[name] {
|
||||
_, err = readArguments(name, args)
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) {
|
||||
t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil {
|
||||
t.Error("a switch took a word that is neither true nor false, and would have read it as false")
|
||||
}
|
||||
if _, err := argvFor("node", map[string]any{"node": 7}); err == nil {
|
||||
t.Error("a number was taken as a machine's name, or as no machine")
|
||||
}
|
||||
if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" {
|
||||
t.Errorf("a switch given as JSON true: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The push that was the cause: a machine named is that machine; none named is the whole mesh, and
|
||||
// naming one beside behind is refused rather than one of the two guessed.
|
||||
func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) {
|
||||
argv, err := argvFor("push", map[string]any{"node": "g1"})
|
||||
if err != nil || strings.Join(argv, " ") != "push g1 --wait 0" {
|
||||
t.Fatalf("a named push: %v %v", argv, err)
|
||||
}
|
||||
for _, args := range []map[string]any{{}, {"behind": "true"}} {
|
||||
argv, err := argvFor("push", args)
|
||||
if err != nil || strings.Join(argv, " ") != "push --behind --wait 0" {
|
||||
t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err)
|
||||
}
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true"}); err == nil ||
|
||||
!strings.Contains(err.Error(), `"behind"`) {
|
||||
t.Fatalf("a named push with behind was taken: %v", err)
|
||||
}
|
||||
if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) {
|
||||
t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// commandFlags is every flag set this package's commands parse, by the name the set is made with,
|
||||
// and the flags defined on it — read from the source, so a flag added to a command is seen here
|
||||
// without anyone remembering to.
|
||||
func commandFlags(t *testing.T) map[string][]string {
|
||||
t.Helper()
|
||||
files, err := filepath.Glob("*.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fset := token.NewFileSet()
|
||||
out := map[string][]string{}
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f, "_test.go") {
|
||||
continue
|
||||
}
|
||||
file, err := parser.ParseFile(fset, f, nil, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, decl := range file.Decls {
|
||||
fn, ok := decl.(*ast.FuncDecl)
|
||||
if !ok || fn.Body == nil {
|
||||
continue
|
||||
}
|
||||
sets := map[string]string{} // variable → the set's name
|
||||
ast.Inspect(fn.Body, func(n ast.Node) bool {
|
||||
if assign, ok := n.(*ast.AssignStmt); ok && len(assign.Lhs) == 1 && len(assign.Rhs) == 1 {
|
||||
if call, ok := assign.Rhs[0].(*ast.CallExpr); ok && isSelector(call.Fun, "flag", "NewFlagSet") {
|
||||
if id, ok := assign.Lhs[0].(*ast.Ident); ok {
|
||||
if name, ok := stringLit(call.Args[0]); ok {
|
||||
sets[id.Name] = name
|
||||
out[name] = append(out[name], []string{}...)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
call, ok := n.(*ast.CallExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
recv, ok := sel.X.(*ast.Ident)
|
||||
if !ok || sets[recv.Name] == "" {
|
||||
return true
|
||||
}
|
||||
arg := 0
|
||||
switch sel.Sel.Name {
|
||||
case "Bool", "String", "Int", "Int64", "Uint", "Uint64", "Float64", "Duration", "Func", "BoolFunc", "TextVar":
|
||||
case "BoolVar", "StringVar", "IntVar", "Int64Var", "UintVar", "Uint64Var", "Float64Var", "DurationVar", "Var":
|
||||
arg = 1
|
||||
default:
|
||||
return true
|
||||
}
|
||||
if arg < len(call.Args) {
|
||||
if flagName, ok := stringLit(call.Args[arg]); ok {
|
||||
out[sets[recv.Name]] = append(out[sets[recv.Name]], flagName)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isSelector(e ast.Expr, pkg, name string) bool {
|
||||
sel, ok := e.(*ast.SelectorExpr)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
id, ok := sel.X.(*ast.Ident)
|
||||
return ok && id.Name == pkg && sel.Sel.Name == name
|
||||
}
|
||||
|
||||
func stringLit(e ast.Expr) (string, bool) {
|
||||
lit, ok := e.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
return "", false
|
||||
}
|
||||
s, err := strconv.Unquote(lit.Value)
|
||||
return s, err == nil
|
||||
}
|
||||
|
||||
// accountedFlags are the flags of a verb's command that are not an argument of the same name:
|
||||
// carried by an argument named otherwise ("=argument"), or set by the verb itself, or withheld from
|
||||
// the named verb on purpose — each with why. `command` reaches every flag of the binary regardless.
|
||||
var accountedFlags = map[string]map[string]string{
|
||||
"status": {"json": "set by the verb: the answer is data"},
|
||||
"seats": {"json": "set by the verb: the answer is data"},
|
||||
"queue": {"json": "not set: the verb answers the table a person reads"},
|
||||
"plan": {"json": "set by the verb unless files is asked"},
|
||||
"push": {"wait": "set by the verb to 0: a tool call cannot hold a connection for a whole apply"},
|
||||
"build": {
|
||||
"wait": "set by the verb to 0: the id follows the build (issue 176)",
|
||||
"self": "set by the verb from the repository's form: a path on the forge, or a URL",
|
||||
"dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it",
|
||||
"behind": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
"on": "withheld: the named verb builds one named repository; `command` reaches the rest",
|
||||
},
|
||||
"builds": {"n": "=limit"},
|
||||
"plans": {"n": "=limit", "what-if": "=repository"},
|
||||
}
|
||||
|
||||
// **Every flag of the command a verb runs is in the verb's schema, or accounted for here.** Derived
|
||||
// from the source, so a flag added to a command — or a verb added whose command takes flags —
|
||||
// fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And
|
||||
// a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody
|
||||
// reads.
|
||||
func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) {
|
||||
flags := commandFlags(t)
|
||||
if len(flags["push"]) == 0 || len(flags["plan"]) == 0 {
|
||||
t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags)
|
||||
}
|
||||
reached := map[string]map[string]bool{} // verb → flag sets its command lines reach
|
||||
served := servedSchemas(t)
|
||||
for name, v := range served {
|
||||
if inProcess[name] || name == "command" {
|
||||
continue
|
||||
}
|
||||
names, switches := declaredArguments(v)
|
||||
for _, subset := range subsetsOf(names) {
|
||||
argv, err := argvFor(name, argumentsFor(subset, switches))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// The flag set is named by the longest run of leading words that names one.
|
||||
var words []string
|
||||
for _, w := range argv {
|
||||
if strings.HasPrefix(w, "-") {
|
||||
break
|
||||
}
|
||||
words = append(words, w)
|
||||
}
|
||||
for n := len(words); n > 0; n-- {
|
||||
if _, has := flags[strings.Join(words[:n], " ")]; has {
|
||||
if reached[name] == nil {
|
||||
reached[name] = map[string]bool{}
|
||||
}
|
||||
reached[name][strings.Join(words[:n], " ")] = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
used := map[string]map[string]bool{}
|
||||
verbs := make([]string, 0, len(reached))
|
||||
for verb := range reached {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Strings(verbs)
|
||||
for _, verb := range verbs {
|
||||
declared, _ := declaredArguments(served[verb])
|
||||
for set := range reached[verb] {
|
||||
if used[set] == nil {
|
||||
used[set] = map[string]bool{}
|
||||
}
|
||||
for _, flagName := range flags[set] {
|
||||
why, accounted := accountedFlags[set][flagName]
|
||||
switch {
|
||||
case accounted && strings.HasPrefix(why, "="):
|
||||
used[set][flagName] = true
|
||||
if !contains(declared, strings.TrimPrefix(why, "=")) {
|
||||
t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare",
|
||||
verb, flagName, set, strings.TrimPrefix(why, "="))
|
||||
}
|
||||
case accounted:
|
||||
used[set][flagName] = true
|
||||
case contains(declared, flagName):
|
||||
default:
|
||||
t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+
|
||||
"or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for set, fs := range accountedFlags {
|
||||
for flagName := range fs {
|
||||
if !used[set][flagName] {
|
||||
t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Every verb's required arguments are properties of its schema: a schema that requires what it
|
||||
// does not describe is the uncallable verb of issue 244 from the other side.
|
||||
func TestEveryRequiredArgumentIsDescribed(t *testing.T) {
|
||||
for name, v := range servedSchemas(t) {
|
||||
names, _ := declaredArguments(v)
|
||||
for k := range sampleArguments(v) {
|
||||
if !contains(names, k) {
|
||||
t.Errorf("%s requires %q and does not describe it", name, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// `builds` given a build's id reads that build's log from the bus rather than listing builds
|
||||
// (novox/hq ADR 0157).
|
||||
func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) {
|
||||
argv, err := argvFor("builds", map[string]any{"log": "build-17"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "builds --log build-17" {
|
||||
t.Fatalf("builds with a log id became %q", strings.Join(argv, " "))
|
||||
}
|
||||
}
|
||||
|
||||
// The build tool takes a repository as a URL or as its path on the forge holding the git seat, and
|
||||
// says which it was given, so the command reads the path as a seat source rather than handing it to
|
||||
// git as written (novox/hq issue 176). And it never waits: the id follows the build.
|
||||
func TestTheBuildToolTellsAForgePathFromAURL(t *testing.T) {
|
||||
argv, _ := argvFor("build", map[string]any{"repository": "novox/mesh-catalog", "path": "modules/x"})
|
||||
if line := strings.Join(argv, " "); !strings.Contains(line, "--self") || !strings.Contains(line, "--wait 0") {
|
||||
t.Fatalf("a forge path is a seat source, not waited for; got %q", line)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "https://example.tld/o/r.git"})
|
||||
if line := strings.Join(argv, " "); strings.Contains(line, "--self") {
|
||||
t.Fatalf("a URL is cloned as given; got %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// `rotate` is one verb with two shapes (ADR 0114, issue 180): a pair credential by provision, or a
|
||||
// module's own secret by machine, module and name.
|
||||
func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) {
|
||||
argv, _ := argvFor("rotate", map[string]any{"provision": "postgres-database", "consumer": "ace"})
|
||||
if strings.Join(argv, " ") != "rotate postgres-database --consumer ace" {
|
||||
t.Fatalf("a pair credential: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("rotate", map[string]any{"node": "ace", "module": "nodered", "secret": "api-token"})
|
||||
if strings.Join(argv, " ") != "secret rotate ace nodered api-token" {
|
||||
t.Fatalf("an own secret: %v", argv)
|
||||
}
|
||||
if _, err := argvFor("rotate", map[string]any{"node": "ace"}); err == nil || !strings.Contains(err.Error(), `"module"`) {
|
||||
t.Fatalf("half an own secret is refused, naming what it lacks: %v", err)
|
||||
}
|
||||
if argv, _ := argvFor("rotate", nil); strings.Join(argv, " ") != "rotate" {
|
||||
t.Fatalf("neither shape falls to the command's usage: %v", argv)
|
||||
}
|
||||
if _, err := argvFor("rotate", map[string]any{"provision": "p", "node": "ace", "module": "m", "secret": "s"}); err == nil {
|
||||
t.Fatal("both shapes at once were taken, and one of them passed over")
|
||||
}
|
||||
}
|
||||
|
||||
// `settings` is `settings set|clear` at a shell, with the values passed inline (novox/hq issue 198).
|
||||
func TestSettingsSetsOrClearsALayer(t *testing.T) {
|
||||
argv, err := argvFor("settings", map[string]any{"module": "dnsmasq", "values": `{"a":1}`, "node": "ace"})
|
||||
if err != nil || strings.Join(argv, " ") != `settings set dnsmasq {"a":1} --node ace` {
|
||||
t.Fatalf("set on a machine: %v %v", argv, err)
|
||||
}
|
||||
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq", "clear": "true"})
|
||||
if strings.Join(argv, " ") != "settings clear dnsmasq" {
|
||||
t.Fatalf("clear for the mesh: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("settings", map[string]any{"module": "dnsmasq"})
|
||||
if strings.Join(argv, " ") != "settings set dnsmasq" {
|
||||
t.Fatalf("a set with no values falls to the command's usage: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// `issue` is `module issue` at a shell: the module and the machine, and nothing that would push. A
|
||||
// module's bus account was mintable only from the controller's command line, so an agent working
|
||||
// through the tools could not finish a rollout that gave a module one (novox/hq issue 191).
|
||||
func TestIssueGivesAModuleItsAccountOnAMachine(t *testing.T) {
|
||||
argv, err := argvFor("issue", map[string]any{"node": "ace", "module": "route-proxy"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Join(argv, " ") != "module issue route-proxy --node ace" {
|
||||
t.Fatalf("issue runs %v", argv)
|
||||
}
|
||||
if _, err := argvFor("issue", map[string]any{"module": "route-proxy"}); err == nil {
|
||||
t.Error("an account was issued without saying which machine reads it")
|
||||
}
|
||||
}
|
||||
|
||||
// A required argument missing is refused in the verb's own words, before anything runs.
|
||||
func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) {
|
||||
if _, err := argvFor("node", map[string]any{}); err == nil || !strings.Contains(err.Error(), `node needs "node"`) {
|
||||
t.Fatalf("node without a machine was accepted: %v", err)
|
||||
}
|
||||
if _, err := argvFor("upgrade", map[string]any{}); err == nil {
|
||||
t.Fatal("a verb the seat does not serve was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A push and a build are sent, not waited for: the asker reads status, or the build's log by its
|
||||
// id, for what happened. A repository given as a forge path is said to be one (issue 176).
|
||||
func TestActsDoNotBlockTheCall(t *testing.T) {
|
||||
argv, _ := argvFor("push", map[string]any{"node": "one"})
|
||||
if strings.Join(argv, " ") != "push one --wait 0" {
|
||||
t.Fatalf("push waits: %v", argv)
|
||||
}
|
||||
argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"})
|
||||
if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" {
|
||||
t.Fatalf("build: %v", argv)
|
||||
}
|
||||
}
|
||||
|
||||
// What `tools` answers is the seats' records, with each verb's schema.
|
||||
func TestToolsAnswersTheSeatsRecords(t *testing.T) {
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(behind) != 0 {
|
||||
t.Fatalf("this build cannot run %v of its own seat's verbs", behind)
|
||||
}
|
||||
if len(handlers) != len(catalogue.ControllerVerbs) {
|
||||
t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs))
|
||||
}
|
||||
answer := seatTools()
|
||||
seats, _ := answer["seats"].([]map[string]any)
|
||||
var found bool
|
||||
for _, s := range seats {
|
||||
if s["seat"] == catalogue.ControllerSeatName {
|
||||
found = true
|
||||
tools, _ := s["tools"].([]map[string]any)
|
||||
if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil {
|
||||
t.Fatalf("the controller seat's tools are not listed in full: %v", tools)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("the mesh-controller seat is not in the listing")
|
||||
}
|
||||
}
|
||||
|
||||
// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it
|
||||
// printed beside the document does not take the document away. The test binary stands in for the
|
||||
// controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests.
|
||||
func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) {
|
||||
jsonVerbs["-test.run"] = true
|
||||
t.Cleanup(func() { delete(jsonVerbs, "-test.run") })
|
||||
answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !answer.OK {
|
||||
t.Fatalf("the command failed: %s", answer.Output)
|
||||
}
|
||||
if !strings.Contains(answer.Output, "PASS") {
|
||||
t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output)
|
||||
}
|
||||
}
|
||||
|
||||
// `command` is the generic verb: the command line as given, split as a shell would, nothing added —
|
||||
// so an operator's `node account g14 jochen` is one call through the console rather than a shell on
|
||||
// the control node (novox/hq ADR 0154, ADR 0175).
|
||||
func TestCommandRunsTheLineAsGiven(t *testing.T) {
|
||||
argv, err := argvFor("command", map[string]any{"command": "node account g14 jochen"})
|
||||
if err != nil || strings.Join(argv, " ") != "node account g14 jochen" {
|
||||
t.Fatalf("a plain line: %v %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("command", map[string]any{"command": `settings set dnsmasq '{"a": "b c"}' --node ace`})
|
||||
if err != nil || len(argv) != 6 || argv[3] != `{"a": "b c"}` {
|
||||
t.Fatalf("a quoted word stays one word: %q %v", argv, err)
|
||||
}
|
||||
argv, err = argvFor("command", map[string]any{"command": `node add "the box" --adopted`})
|
||||
if err != nil || len(argv) != 4 || argv[2] != "the box" {
|
||||
t.Fatalf("double quotes group: %q %v", argv, err)
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": " "}); err == nil {
|
||||
t.Fatal("an empty line was accepted")
|
||||
}
|
||||
if _, err := argvFor("command", map[string]any{"command": `node "unclosed`}); err == nil {
|
||||
t.Fatal("an unclosed quote was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// A verb in the row that this binary cannot run does not take the control plane off the bus: the
|
||||
// rest are served, the unknown one answers the reason, and the start-up names it (novox/hq ADR
|
||||
// 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by
|
||||
// a person running the binary by hand — the push that would have repaired it needs the control
|
||||
// plane that was down.
|
||||
func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) {
|
||||
seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName)
|
||||
if !known {
|
||||
t.Fatal("no controller seat")
|
||||
}
|
||||
// The row as a newer control plane would have written it: every verb this build knows, and one
|
||||
// it does not.
|
||||
widened := seat
|
||||
widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...),
|
||||
catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"})
|
||||
rows := catalogue.DefaultSeats()
|
||||
for i := range rows {
|
||||
if rows[i].Name == catalogue.ControllerSeatName {
|
||||
rows[i] = widened
|
||||
}
|
||||
}
|
||||
catalogue.UseSeats(rows)
|
||||
t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) })
|
||||
|
||||
handlers, behind, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatalf("a row with one unknown verb refused to serve at all: %v", err)
|
||||
}
|
||||
if len(behind) != 1 || behind[0] != "teleport" {
|
||||
t.Fatalf("the verbs this build cannot run were reported as %v", behind)
|
||||
}
|
||||
if len(handlers) != len(widened.Serves) {
|
||||
t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves))
|
||||
}
|
||||
for _, known := range []string{"status", "nodes", "push"} {
|
||||
if handlers[known] == nil {
|
||||
t.Errorf("%s is not served although this build knows it", known)
|
||||
}
|
||||
}
|
||||
_, err = handlers["teleport"](context.Background(), nil)
|
||||
if err == nil {
|
||||
t.Fatal("the unknown verb answered as though it had run")
|
||||
}
|
||||
for _, want := range []string{"teleport", "cannot run it", "behind"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the answer does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON,
|
||||
// as status and seats do, so nothing parses a printed column.
|
||||
func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) {
|
||||
for verb, want := range map[string]string{"nodes": "[node list --json]", "modules": "[module list --json]"} {
|
||||
argv, err := argvFor(verb, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(argv) != want {
|
||||
t.Errorf("%s runs %v, want %s", verb, argv, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and
|
||||
// queue it serves it on, with the seat's own description and schema, in NATS's services format.
|
||||
func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) {
|
||||
handlers, _, err := seatToolHandlers()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info := seatAnnouncement(handlers)
|
||||
if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" {
|
||||
t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity)
|
||||
}
|
||||
if len(info.Endpoints) != len(handlers) {
|
||||
t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers))
|
||||
}
|
||||
for _, e := range info.Endpoints {
|
||||
verb := e.Metadata["tool"]
|
||||
if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb {
|
||||
t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb)
|
||||
}
|
||||
if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata)
|
||||
}
|
||||
if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName {
|
||||
t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup)
|
||||
}
|
||||
if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" {
|
||||
t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/secrets"
|
||||
)
|
||||
|
||||
// secretCommand gives the mesh a value it must carry and could not have invented.
|
||||
//
|
||||
// **Every other secret in this mesh is one the mesh made** — generated, sealed to the machine that
|
||||
// will use it, and never readable again. That is right for something coming into existence, and
|
||||
// wrong for something that already exists: a database created last year has the password it was
|
||||
// created with, and generating a new one puts 32 random bytes where a working credential was.
|
||||
// The machine applies it, reports success, and whatever reads it fails to authenticate somewhere
|
||||
// else entirely — with the mesh insisting the secret was delivered, which it was.
|
||||
//
|
||||
// So this is the entry point for **adopting** something already running. The store has carried
|
||||
// the distinction since the beginning: a module secret records whether it was `made` or
|
||||
// `accepted`, and refuses to invent a replacement for the second. Nothing until now could write
|
||||
// one, so the only accepted secret in the mesh was the broker account issued to a build machine.
|
||||
//
|
||||
// The value is sealed on the way in and the plaintext discarded, exactly as a generated one is.
|
||||
// **The only difference between the two is where the value came from.**
|
||||
func secretCommand(ctx context.Context, args []string) error {
|
||||
if len(args) == 0 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
switch args[0] {
|
||||
case "accept":
|
||||
case "rotate":
|
||||
return secretRotate(ctx, args[1:])
|
||||
case "recover":
|
||||
return secretRecover(ctx, args[1:])
|
||||
case "export":
|
||||
return secretExport(ctx, args[1:])
|
||||
default:
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
rest, flags := split(args[1:])
|
||||
set := flag.NewFlagSet("secret accept", flag.ContinueOnError)
|
||||
from := set.String("from", "",
|
||||
"read the value from this file instead of asking (use - for standard input)")
|
||||
provider := set.String("provider", "",
|
||||
"the node providing <name>: the value becomes the PAIR credential between <module> on <node> "+
|
||||
"and that provider, sealed to both — the vault's operator-delivered secret (ADR 0092)")
|
||||
local := set.String("local", "",
|
||||
"with --provider: the name the credential goes by inside <module>, where its manifest keeps "+
|
||||
"several for <name> (ADR 0094)")
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
|
||||
value, err := valueFor(node, module, name, *from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
value = asSupplied(value)
|
||||
if value == "" {
|
||||
return errors.New("there is nothing to seal")
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
if *provider != "" {
|
||||
// Into the pair, not into the module's own secrets: what the provider is asked to create
|
||||
// and what the consumer reads are the same value, and neither end can be told a different
|
||||
// one later without the other (novox/hq 04-ISSUES/070).
|
||||
if err := open.inventory.AcceptSecretForPair(ctx, name, node, module, *provider, *local, value); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s now holds %q from %s%s, sealed to both machines.\n", module, node, name, *provider, asLocal(*local))
|
||||
fmt.Printf(" the mesh cannot read it back, will not replace it with one of its own, and will not rotate it\n")
|
||||
fmt.Printf(" run `push %s` and `push %s` to send it\n", *provider, node)
|
||||
return nil
|
||||
}
|
||||
if err := open.inventory.AcceptSecretForModule(ctx, node, module, name, value); err != nil {
|
||||
return err
|
||||
}
|
||||
// Not printed back, and there is nowhere it could be printed from: it is sealed to that
|
||||
// machine and the mesh cannot read it again.
|
||||
fmt.Printf("%s on %s now holds %q, sealed to that machine.\n", module, node, name)
|
||||
fmt.Printf(" the mesh cannot read it back, and will not replace it with one of its own\n")
|
||||
fmt.Printf(" run `push %s` to send it\n", node)
|
||||
return nil
|
||||
}
|
||||
|
||||
const secretUsage = "secret rotate <node> <module> <name>\n" +
|
||||
"secret accept <node> <module> <name> [--from <file>] [--provider <node> [--local <name>]]\n" +
|
||||
"secret recover <node> <module> <name> --key <operator-key> [--out <file>] [--from-export <file>] [--provider <node>]\n" +
|
||||
"secret export [--out <file>]"
|
||||
|
||||
// secretRecover is break-glass: a secret opened with the operator's key, written to a file.
|
||||
//
|
||||
// **The mesh cannot show a secret back, and this does not make it able to.** What is opened here
|
||||
// is the copy sealed to the operator key (novox/hq ADR 0085, amended); the mesh holds that blob and
|
||||
// no key for it, and this program holds the key for the length of the call and no blob until given
|
||||
// one. Recovery needs both, which is what keeps the sealing meaningful.
|
||||
//
|
||||
// The value goes to a file at 0600, never to the terminal unless asked for with `--out -` — the
|
||||
// source mesh's secret tools were written after a secret was printed into a transcript, and that
|
||||
// rule is theirs. `--from-export` reads the blob from a file `secret export` wrote, so recovery
|
||||
// works with the store gone, which is the case it exists for.
|
||||
func secretRecover(ctx context.Context, args []string) error {
|
||||
rest, flags := split(args)
|
||||
set := flag.NewFlagSet("secret recover", flag.ContinueOnError)
|
||||
keyFile := set.String("key", "", "the operator's private key, from `operator key make`")
|
||||
out := set.String("out", "", "where to write the value (0600); - for standard output. Default <node>.<module>.<name>.secret")
|
||||
fromExport := set.String("from-export", "", "read the sealed copy from this `secret export` file instead of the store")
|
||||
provider := set.String("provider", "", "for a pair credential held from more than one provider: which one")
|
||||
local := set.String("local", "", "for a pair credential the module keeps under a local name (ADR 0094): which one")
|
||||
if err := set.Parse(flags); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rest) != 3 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
private, err := readPrivateKey(*keyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var kept inventory.Kept
|
||||
if *fromExport != "" {
|
||||
kept, err = keptFromExport(*fromExport, node, module, name, *provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
kept, err = open.inventory.KeptSecret(ctx, node, module, name, *provider, *local)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
value, err := secrets.Open(private, kept.Sealed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s on %s: %q is sealed to operator key %s, and that key does not open it: %w",
|
||||
module, node, name, secrets.Fingerprint(kept.Key), err)
|
||||
}
|
||||
if *out == "-" {
|
||||
_, err := os.Stdout.Write(append(value, '\n'))
|
||||
return err
|
||||
}
|
||||
path := *out
|
||||
if path == "" {
|
||||
path = node + "." + module + "." + name + ".secret"
|
||||
}
|
||||
if err := writeNew(path, value); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s on %s: %q recovered to %s (0600) — %d bytes, origin %s\n",
|
||||
module, node, name, path, len(value), kept.Origin)
|
||||
return nil
|
||||
}
|
||||
|
||||
// An export is what a person keeps beside the operator key — the catalogue's shape, so the vault
|
||||
// keeps the same document on its disk (Manifest.Keeps).
|
||||
type export = catalogue.KeptExport
|
||||
|
||||
func secretExport(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("secret export", flag.ContinueOnError)
|
||||
out := set.String("out", "", "where to write the export (0600); - or empty for standard output")
|
||||
if err := set.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
key, err := inv.OperatorKey(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if key == "" {
|
||||
return errors.New("the mesh has no operator key, so nothing is sealed to one; `operator key make` and `operator key set` first")
|
||||
}
|
||||
doc, err := inv.OperatorExport(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := json.MarshalIndent(doc, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = append(body, '\n')
|
||||
if *out == "" || *out == "-" {
|
||||
_, err := os.Stdout.Write(body)
|
||||
return err
|
||||
}
|
||||
// Replaced whole, and made 0600 whether or not it existed: an export is ciphertext and a public
|
||||
// key, but it is also the list of every secret the mesh has, and a file left at an earlier mode
|
||||
// while the command says 0600 is a lie in the one place a person checks.
|
||||
if err := writeReplacing(*out, body); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%d secret(s) exported to %s (0600), sealed to operator key %s — ciphertext, keep it with the key\n",
|
||||
len(doc.Kept), *out, doc.Fingerprint)
|
||||
if len(doc.EarlierKey) > 0 {
|
||||
fmt.Printf(" %d secret(s) are sealed to an EARLIER operator key: recoverable with that key only\n", len(doc.EarlierKey))
|
||||
}
|
||||
if len(doc.Unrecoverable) > 0 {
|
||||
fmt.Printf(" %d secret(s) are NOT in it: made before the mesh had an operator key\n", len(doc.Unrecoverable))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeNew writes a file that must not exist yet, atomically: create-exclusive, 0600. A check
|
||||
// followed by a write is a window in which a key somebody still needs can be overwritten.
|
||||
func writeNew(path string, content []byte) error {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
if os.IsExist(err) {
|
||||
return fmt.Errorf("%s already exists; not overwriting it", path)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(content); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// writeReplacing writes a file whole, creating or truncating it, and leaves it at 0600 either way.
|
||||
func writeReplacing(path string, content []byte) error {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(content); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Chmod(0o600); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
func keptFromExport(path, node, module, name, provider string) (inventory.Kept, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return inventory.Kept{}, err
|
||||
}
|
||||
var e export
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return inventory.Kept{}, fmt.Errorf("%s is not a secret export: %w", path, err)
|
||||
}
|
||||
// Sealed to the current key or to an earlier one: both are copies the given key might open,
|
||||
// and Open says which. Not the unrecoverable list, which holds no copy at all.
|
||||
var found []inventory.Kept
|
||||
for _, k := range append(append([]inventory.Kept{}, e.Kept...), e.EarlierKey...) {
|
||||
if k.Node == node && k.Module == module && k.Name == name && (provider == "" || k.Provider == provider) {
|
||||
found = append(found, k)
|
||||
}
|
||||
}
|
||||
switch len(found) {
|
||||
case 0:
|
||||
return inventory.Kept{}, fmt.Errorf("%s holds no copy of %s's %q on %s", path, module, name, node)
|
||||
case 1:
|
||||
return found[0], nil
|
||||
default:
|
||||
providers := make([]string, 0, len(found))
|
||||
for _, f := range found {
|
||||
providers = append(providers, f.Provider)
|
||||
}
|
||||
return inventory.Kept{}, fmt.Errorf("%s holds %s's %q on %s from more than one provider (%s); say which with --provider",
|
||||
path, module, name, node, strings.Join(providers, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// split separates what this command is about from how it was asked.
|
||||
//
|
||||
// **Because the standard library stops parsing at the first non-flag argument.** With the
|
||||
// positionals first — which is the order that reads correctly — everything after them is left
|
||||
// sitting in the arguments, so `secret accept a b c --from -` arrives as five positionals and the
|
||||
// flag is never seen. The host's own parser carries the same note, and the fault it names is
|
||||
// worse than this one: there, a flag somebody passed was silently ignored and the command
|
||||
// succeeded anyway.
|
||||
func split(args []string) (positional, flags []string) {
|
||||
for i, arg := range args {
|
||||
if strings.HasPrefix(arg, "-") {
|
||||
return args[:i], args[i:]
|
||||
}
|
||||
}
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// asSupplied is the value with its line ending removed and nothing else.
|
||||
//
|
||||
// **A file has a trailing newline and a password does not**, so the ending goes — a credential
|
||||
// wrong by one byte fails in a way nobody connects to how it was supplied.
|
||||
//
|
||||
// **And only the ending.** Trimming both ends is the obvious thing and it is wrong: a password
|
||||
// chosen with a leading space is one the mesh would then deliver as a different password, silently,
|
||||
// with the operator certain they had supplied it correctly.
|
||||
func asSupplied(raw string) string {
|
||||
return strings.TrimRight(raw, "\r\n")
|
||||
}
|
||||
|
||||
// valueFor gets the secret without putting it somewhere it can be read afterwards.
|
||||
//
|
||||
// **Not an argument, and there is no flag that takes one.** A value on the command line is in the
|
||||
// shell's history, in the process list for as long as it runs, and in whatever collects either.
|
||||
// The paths here are a file the operator already has, or a prompt that does not echo — the same
|
||||
// two ways a model-access key is supplied (novox/hq ADR 0024).
|
||||
func valueFor(node, module, name, from string) (string, error) {
|
||||
switch {
|
||||
case from == "-":
|
||||
body, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
|
||||
case from != "":
|
||||
body, err := os.ReadFile(from)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(body), nil
|
||||
|
||||
default:
|
||||
// The same path a model-access key takes, and for the same reason: a value given as an
|
||||
// argument is in the shell's history and in the process list. Read from standard input,
|
||||
// echoed nowhere by this program.
|
||||
fmt.Fprintf(os.Stderr,
|
||||
"reading %s's %q for %s from standard input; it is not echoed anywhere\n",
|
||||
module, name, node)
|
||||
line, err := bufio.NewReader(os.Stdin).ReadString('\n')
|
||||
if err != nil && line == "" {
|
||||
return "", fmt.Errorf("nothing was given on standard input: %w", err)
|
||||
}
|
||||
return line, nil
|
||||
}
|
||||
}
|
||||
|
||||
// secretRotate makes a module's own secret anew and sends the machine, so the module starts again on
|
||||
// the new value (novox/hq ADR 0114, issue 180). A pair credential rotates with `rotate <provision>`;
|
||||
// this is the secret with one party. Said in the log with who asked and when, never the value.
|
||||
func secretRotate(ctx context.Context, args []string) error {
|
||||
rest, _ := split(args)
|
||||
if len(rest) != 3 {
|
||||
return errors.New(secretUsage)
|
||||
}
|
||||
node, module, name := rest[0], rest[1], rest[2]
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
if err := open.inventory.RotateModuleSecret(ctx, node, module, name); err != nil {
|
||||
var refused inventory.ErrNotRotatable
|
||||
if errors.As(err, &refused) {
|
||||
return fmt.Errorf("not rotated: %s", refused.Why)
|
||||
}
|
||||
return err
|
||||
}
|
||||
fmt.Printf("rotated %q of %s on %s at %s, asked by %s; the value is sealed and not shown\n",
|
||||
name, module, node, time.Now().UTC().Format(time.RFC3339), whoAsked())
|
||||
// A shared credential (ADR 0158) has as many holders as the provision has consumers, and all
|
||||
// of them are sent in one act, so no machine is left reading a value the provider no longer takes.
|
||||
machines, err := open.inventory.SharedHolders(ctx, node, module, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(machines) == 0 {
|
||||
machines = []string{node}
|
||||
}
|
||||
if len(machines) == 1 {
|
||||
fmt.Printf("sending %s, so %s starts again on the new value:\n", node, module)
|
||||
} else {
|
||||
fmt.Printf("shared with every consumer; sending %s together:\n", strings.Join(machines, ", "))
|
||||
}
|
||||
if err := sendTo(ctx, open, machines); err != nil {
|
||||
return fmt.Errorf("%w\n\nThe new value is sealed and not yet delivered; what runs keeps the old "+
|
||||
"one until the machines next apply. Fix the cause and run `push --behind`", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// whoAsked names the caller for the log: the account the command runs as, which for a tool call
|
||||
// through the console is the mesh's own.
|
||||
func whoAsked() string {
|
||||
if u := os.Getenv("SUDO_USER"); u != "" {
|
||||
return u
|
||||
}
|
||||
if u := os.Getenv("USER"); u != "" {
|
||||
return u
|
||||
}
|
||||
return "the mesh"
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A file has a trailing newline and a password does not.
|
||||
//
|
||||
// The failure this prevents is the worst kind to diagnose: the credential is delivered, the
|
||||
// machine applies it, everything reports success, and authentication fails one byte from correct
|
||||
// somewhere else entirely.
|
||||
func TestATrailingNewlineIsNotPartOfTheSecret(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "password")
|
||||
if err := os.WriteFile(path, []byte("the-database-password\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := valueFor("anchor", "umami", "database", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asSupplied(got) != "the-database-password" {
|
||||
t.Fatalf("read %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A password may contain spaces, and they are the operator's.
|
||||
//
|
||||
// Trimming both ends is the obvious thing and it is wrong: a value chosen with a leading space is
|
||||
// a value the mesh would silently deliver as a different one.
|
||||
func TestOnlyLineEndingsAreRemoved(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "password")
|
||||
if err := os.WriteFile(path, []byte(" spaces matter \n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := valueFor("anchor", "umami", "database", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asSupplied(got) != " spaces matter " {
|
||||
t.Fatalf("the value was altered beyond its line ending: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A file that is not there is said plainly, rather than becoming an empty secret.
|
||||
func TestAMissingFileIsRefused(t *testing.T) {
|
||||
if _, err := valueFor("anchor", "umami", "database",
|
||||
filepath.Join(t.TempDir(), "absent")); err == nil {
|
||||
t.Fatal("a missing file produced a value")
|
||||
}
|
||||
}
|
||||
|
||||
// The command refuses what it cannot act on, rather than acting on part of it.
|
||||
func TestTheArgumentsAreRequired(t *testing.T) {
|
||||
for _, args := range [][]string{
|
||||
{},
|
||||
{"accept"},
|
||||
{"accept", "anchor"},
|
||||
{"accept", "anchor", "umami"},
|
||||
{"give", "anchor", "umami", "database"},
|
||||
} {
|
||||
if err := secretCommand(t.Context(), args); err == nil {
|
||||
t.Errorf("%v was accepted", args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The invocation that actually gets used, which the first version of these tests never tried.
|
||||
//
|
||||
// Every case here was a rejection, so the command was broken in the one way that matters — it
|
||||
// refused what it is for — and the tests were green. The lab found it at the first call.
|
||||
func TestTheArgumentsAndTheFlagAreBothSeen(t *testing.T) {
|
||||
rest, flags := split([]string{"anchor", "umami", "database", "--from", "-"})
|
||||
assert(t, len(rest) == 3, "the three positionals were not kept: %v", rest)
|
||||
assert(t, len(flags) == 2, "the flag was not separated: %v", flags)
|
||||
|
||||
// And with no flag at all, which is the interactive form.
|
||||
rest, flags = split([]string{"anchor", "umami", "database"})
|
||||
assert(t, len(rest) == 3 && len(flags) == 0, "%v / %v", rest, flags)
|
||||
|
||||
// A flag before the positionals still works, because somebody will write it that way.
|
||||
rest, flags = split([]string{"--from", "/tmp/x"})
|
||||
assert(t, len(rest) == 0 && len(flags) == 2, "%v / %v", rest, flags)
|
||||
}
|
||||
|
||||
// And the wiring, not just the helper.
|
||||
//
|
||||
// Testing `split` alone left the command able to ignore it entirely — removing the call changed
|
||||
// no test. This reaches secretCommand: with a `--from` naming a file that is not there, the
|
||||
// complaint must be about the file. A complaint about usage would mean the flag was never seen.
|
||||
func TestTheCommandItselfSeesTheFlag(t *testing.T) {
|
||||
err := secretCommand(t.Context(),
|
||||
[]string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"})
|
||||
if err == nil {
|
||||
t.Fatal("a missing file was accepted")
|
||||
}
|
||||
if strings.Contains(err.Error(), "secret accept <node>") {
|
||||
t.Fatalf("the command did not see its flag and complained about usage instead: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assert(t *testing.T, ok bool, format string, args ...any) {
|
||||
t.Helper()
|
||||
if !ok {
|
||||
t.Fatalf(format, args...)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// sendable is a declaration as a machine is sent it: its resources and, for an adopted node, its
|
||||
// mode and which modules were taken on it (novox/hq ADR 0100).
|
||||
//
|
||||
// **Body is the only place the envelope is marshalled.** It was written by hand at every send site,
|
||||
// in the digest the mesh compares, and in `plan --json`; a key added at one and not another would
|
||||
// make a machine look out of date for ever, or send something `plan` never showed.
|
||||
type sendable struct {
|
||||
Resources []map[string]any
|
||||
// Sequence orders this send against every other to the same node: one higher each time, taken
|
||||
// under the node's hold just before the body is made (novox/hq 04-ISSUES/107). Zero is not sent
|
||||
// at all, which a host reads as "no order claimed" — the shape of every declaration before this.
|
||||
Sequence int64
|
||||
// Adoption is nil for a converged node, and then the body is byte for byte what it was before
|
||||
// adoption existed: an older host parses the envelope strictly and would refuse the key.
|
||||
Adoption *adoptionEnvelope
|
||||
|
||||
// Received and Mesh are not sent in the declaration. They are what this machine's memberships
|
||||
// are issued with on the bus (novox/hq ADR 0167): each module's received contributions, from
|
||||
// the same composition as its received files, and every machine's private-network address.
|
||||
Received map[string]map[string][]catalogue.Contribution
|
||||
Mesh []string
|
||||
// BusUsers is the bus's user list this declaration carries, empty for every machine but the one
|
||||
// holding the bus; not sent apart from the file it is in. Its digest is recorded once sent, so
|
||||
// whether that machine must go first is read from the list alone (novox/hq issue 249).
|
||||
BusUsers string
|
||||
// LeftOut is every module of the machine's set left out of this declaration because a stored
|
||||
// setting cannot compose with its definition (novox/hq ADR 0163, rule 6), sorted. The host
|
||||
// keeps that module's held things and touches none of its containers; a machine is told
|
||||
// everything or nothing about what it IS told, and what it is not told is said. Absent from
|
||||
// the body when empty, so a declaration that leaves nothing out is byte for byte what it was.
|
||||
LeftOut []string
|
||||
// leftOutWhy is why each was, for push and plan to say; never on the wire.
|
||||
leftOutWhy map[string]string
|
||||
// Builds is the build of each module this declaration carries — module to the commit its build
|
||||
// was made from — recorded with the send and never on the wire (novox/hq issue 259, ADR 0221).
|
||||
// Composed only on the send path; nil records that it is not known.
|
||||
Builds map[string]string
|
||||
}
|
||||
|
||||
// adoptionEnvelope is what an adopted node is told about its mode. Taken is every module taken on
|
||||
// it that it runs; Untaken is, for every module it runs that is not taken, the ids of every one of
|
||||
// that module's resources — what the host keeps as found until the module is taken (ADR 0103). Ids
|
||||
// rather than a rule to split them by, because a module's name may contain a dot.
|
||||
type adoptionEnvelope struct {
|
||||
Taken []string `json:"taken"`
|
||||
Untaken map[string][]string `json:"untaken,omitempty"`
|
||||
}
|
||||
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
if s.Adoption != nil {
|
||||
envelope["adoption"] = s.Adoption
|
||||
}
|
||||
if s.Sequence > 0 {
|
||||
envelope["sequence"] = s.Sequence
|
||||
}
|
||||
if len(s.LeftOut) > 0 {
|
||||
envelope["left_out"] = s.LeftOut
|
||||
}
|
||||
// An empty declaration is deliberate here — the node owns nothing the mesh put there
|
||||
// (novox/hq issue 127) — and the host refuses an empty body unless it is told the emptiness
|
||||
// is meant, so a truncated or mis-composed body is never mistaken for "own nothing".
|
||||
if len(s.Resources) == 0 {
|
||||
envelope["owns_nothing"] = true
|
||||
}
|
||||
return json.Marshal(envelope)
|
||||
}
|
||||
|
||||
// adoptionOf is the envelope for a node, nil when it is converged.
|
||||
func adoptionOf(ctx context.Context, inv *inventory.Inventory, record inventory.Node,
|
||||
plan catalogue.Resolution, composed catalogue.Composed) (*adoptionEnvelope, error) {
|
||||
if !record.Adopted {
|
||||
return nil, nil
|
||||
}
|
||||
taken, err := inv.Taken(ctx, record.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return adoptionFor(plan, taken, composed), nil
|
||||
}
|
||||
|
||||
// adoptionFor is the envelope computed from what was taken and who owns each resource.
|
||||
//
|
||||
// Every module the node runs that is not taken is untaken — including one pulled in by another
|
||||
// rather than assigned: what is found is kept until its module is taken, whoever put it there.
|
||||
func adoptionFor(plan catalogue.Resolution, taken []string,
|
||||
composed catalogue.Composed) *adoptionEnvelope {
|
||||
isTaken := map[string]bool{}
|
||||
for _, m := range taken {
|
||||
isTaken[m] = true
|
||||
}
|
||||
out := &adoptionEnvelope{Taken: []string{}}
|
||||
runs := map[string]bool{}
|
||||
for _, m := range plan.Modules {
|
||||
runs[m.Module] = true
|
||||
if isTaken[m.Module] {
|
||||
out.Taken = append(out.Taken, m.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out.Taken)
|
||||
for _, r := range composed.Resources {
|
||||
id, _ := r["id"].(string)
|
||||
module, owned := composed.Owner[id]
|
||||
// Every kind, not only files and containers (novox/hq ADR 0103): a directory, a service, a
|
||||
// container mounting what was found and an action run in a held container all reach what
|
||||
// the machine already has. What the mesh declares of its own is never held.
|
||||
if !owned || !runs[module] || isTaken[module] ||
|
||||
strings.HasPrefix(id, catalogue.AdoptionPrefix) {
|
||||
continue
|
||||
}
|
||||
if out.Untaken == nil {
|
||||
out.Untaken = map[string][]string{}
|
||||
}
|
||||
out.Untaken[module] = append(out.Untaken[module], id)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules
|
||||
// were taken on it; a converged node's declaration is byte for byte what it was.
|
||||
|
||||
// helloWeb is a module the predecessor also runs: a page and a server under names it uses.
|
||||
func helloWeb() catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "hello-web", Version: "1",
|
||||
Resources: []map[string]any{
|
||||
{"id": "page", "type": "file", "path": "/var/lib/hello-web/index.html", "content": "hello"},
|
||||
{"id": "server", "type": "container", "name": "hello-web",
|
||||
"image": "registry.example/hello@sha256:" + strings.Repeat("a", 64)},
|
||||
{"id": "served", "type": "directory", "path": "/var/lib/hello-web"},
|
||||
}}
|
||||
}
|
||||
|
||||
// composed is what node would be sent now, as push composes it.
|
||||
func composed(t *testing.T, open *stores, node string) sendable {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(t.Context(), open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared, err := declarationFor(t.Context(), open, node, plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return declared
|
||||
}
|
||||
|
||||
// convergedBefore is the envelope a converged node was sent before adoption existed, captured by
|
||||
// running this same composition at the commit this branch left main (0a39b7d). The mesh here is
|
||||
// aMesh's laptop with the private network taken off it, so nothing in the declaration is random:
|
||||
// what changes this string is a change to what a converged machine is sent, which is the thing an
|
||||
// older host would refuse.
|
||||
//
|
||||
// Re-captured 2026-10-01 (novox/hq issue 177): c978aa7 took `hosts` off every container — a
|
||||
// machine's own resolver knows the mesh's names now — and left this string carrying it, so the
|
||||
// guard failed for a day and nothing ran it. A field an older host never sees is the one change
|
||||
// this guard permits; a field it would refuse is the one it exists to catch.
|
||||
const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}`
|
||||
|
||||
func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, helloWeb())
|
||||
if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared := composed(t, open, "laptop")
|
||||
if declared.Adoption != nil {
|
||||
t.Fatal("a converged node was given an adoption envelope")
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(body) != convergedBefore {
|
||||
t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s",
|
||||
body, convergedBefore)
|
||||
}
|
||||
if bytes.Contains(body, []byte(`"adoption"`)) {
|
||||
t.Fatal("a converged declaration names adoption; an older host would refuse it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, helloWeb())
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
declared := composed(t, open, "anchor")
|
||||
if declared.Adoption == nil {
|
||||
t.Fatal("an adopted node's declaration does not say it is adopted")
|
||||
}
|
||||
untaken := declared.Adoption.Untaken["hello-web"]
|
||||
// Every kind — its directory too (novox/hq ADR 0103).
|
||||
if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server",
|
||||
"hello-web.served"}) {
|
||||
t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption)
|
||||
}
|
||||
if len(declared.Adoption.Taken) != 0 {
|
||||
t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken)
|
||||
}
|
||||
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var envelope map[string]any
|
||||
if err := json.Unmarshal(body, &envelope); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
adoption, _ := envelope["adoption"].(map[string]any)
|
||||
if _, ok := adoption["taken"].([]any); !ok {
|
||||
t.Fatalf("taken is not a list on the wire, even empty: %s", body)
|
||||
}
|
||||
|
||||
// The digest the mesh compares is the digest of what is sent: status and push agree.
|
||||
would, err := wouldSend(ctx, open, mustNodes(t, open))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if would["anchor"] != digestOf(body) {
|
||||
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||
}
|
||||
|
||||
// plan --json prints that same envelope.
|
||||
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||
}
|
||||
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||
}
|
||||
|
||||
// Taking the module moves its resources out of untaken.
|
||||
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
declared = composed(t, open, "anchor")
|
||||
if _, still := declared.Adoption.Untaken["hello-web"]; still {
|
||||
t.Fatalf("a taken module is still untaken: %v", declared.Adoption)
|
||||
}
|
||||
if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) {
|
||||
t.Fatalf("taken is %v", declared.Adoption.Taken)
|
||||
}
|
||||
}
|
||||
|
||||
func mustNodes(t *testing.T, open *stores) []inventory.Node {
|
||||
t.Helper()
|
||||
nodes, err := open.inventory.Nodes(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
|
||||
// stdoutOf is what run printed.
|
||||
func stdoutOf(t *testing.T, run func() error) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := os.Stdout
|
||||
os.Stdout = w
|
||||
done := make(chan string)
|
||||
go func() {
|
||||
all, _ := io.ReadAll(r)
|
||||
done <- string(all)
|
||||
}()
|
||||
runErr := run()
|
||||
os.Stdout = saved
|
||||
w.Close()
|
||||
out := <-done
|
||||
if runErr != nil {
|
||||
t.Fatalf("%v\n%s", runErr, out)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// novox/hq ADR 0100: a port a node was given for the store is where its consumers on other
|
||||
// machines are told to reach it, and a port given for the whole mesh is refused.
|
||||
func TestConsumersAreToldTheGivenPort(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}},
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Guards: []int{5432},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"},
|
||||
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}})
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "store",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var told any
|
||||
for _, n := range plan.Needs {
|
||||
if n.Name == "database" {
|
||||
told = n.Serves["port"]
|
||||
}
|
||||
}
|
||||
if told != 5433 {
|
||||
t.Fatalf("the consumer is told the store is on %v", told)
|
||||
}
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) {
|
||||
t.Fatalf("the store publishes %v", r["ports"])
|
||||
}
|
||||
}
|
||||
|
||||
// A port for the whole mesh is refused where it is set, not stored to refuse every node's
|
||||
// declaration afterwards.
|
||||
if err := open.inventory.SetSettings(ctx, "", "store",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil ||
|
||||
!strings.Contains(err.Error(), "per node") {
|
||||
t.Fatalf("a port given for the whole mesh was not refused: %v", err)
|
||||
}
|
||||
plan, settings, err := planFor(ctx, open, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil {
|
||||
t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store
|
||||
// assigned but not taken is not guarded — its port may still be the predecessor's — and taking it
|
||||
// guards it from the next declaration.
|
||||
func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "store", Version: "1",
|
||||
Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}},
|
||||
Guards: []int{5432},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store",
|
||||
"ports": []any{"5432:5432"},
|
||||
"image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) {
|
||||
t.Fatal("an untaken store is guarded")
|
||||
}
|
||||
if err := open.inventory.Take(ctx, "anchor", "store"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range composed(t, open, "anchor").Resources {
|
||||
if r["id"] == catalogue.GuardID() {
|
||||
if r["content"] != catalogue.AsGuard([]int{5432}) {
|
||||
t.Fatalf("the taken store's guard is:\n%s", r["content"])
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("a taken store is not guarded")
|
||||
}
|
||||
|
||||
// novox/hq ADR 0038 and 0100: a module may publish a port the long way — `2222:22`, because the
|
||||
// machine's own ssh daemon holds 22 — and say it listens on the machine side of that mapping,
|
||||
// which is the number anything reaching it dials. A node moves that port by naming it, and the
|
||||
// number has to reach everything derived from it at once: what the runtime is handed, what an
|
||||
// adopted node is told to open, what its guard refuses, and what a consumer elsewhere dials.
|
||||
//
|
||||
// It reached none of them. The setting was refused outright for naming the machine side — so a
|
||||
// module's port could not be put back where the machine it replaces had it, and, worse, the
|
||||
// node's every push failed for as long as the setting existed.
|
||||
func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "forge", Version: "1",
|
||||
Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}},
|
||||
Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}},
|
||||
Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}},
|
||||
Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge",
|
||||
"ports": []any{"2222:22"},
|
||||
"image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}})
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1",
|
||||
Requires: []string{"git-over-ssh"}})
|
||||
if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "forge"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "anchor", "forge",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
resources := composed(t, open, "anchor").Resources
|
||||
var container, opening map[string]any
|
||||
for _, r := range resources {
|
||||
switch r["id"] {
|
||||
case "forge.server":
|
||||
container = r
|
||||
case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded):
|
||||
opening = r
|
||||
}
|
||||
if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") {
|
||||
t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id)
|
||||
}
|
||||
}
|
||||
if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) {
|
||||
t.Fatalf("the forge's container publishes %v", container["ports"])
|
||||
}
|
||||
if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh {
|
||||
t.Fatalf("no opening for the port this node gave the forge: %v", opening)
|
||||
}
|
||||
var guard map[string]any
|
||||
for _, r := range resources {
|
||||
if r["id"] == catalogue.GuardID() {
|
||||
guard = r
|
||||
}
|
||||
}
|
||||
if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) {
|
||||
t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"])
|
||||
}
|
||||
|
||||
// And the consumer on the other machine dials the same number.
|
||||
plan, _, err := planFor(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var told any
|
||||
for _, n := range plan.Needs {
|
||||
if n.Name == "git-over-ssh" {
|
||||
told = n.Serves["port"]
|
||||
}
|
||||
}
|
||||
if told != 222 {
|
||||
t.Fatalf("the consumer is told the forge answers on %v", told)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) {
|
||||
// The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127).
|
||||
body, err := sendable{}.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if env["owns_nothing"] != true {
|
||||
t.Fatalf("an empty declaration must mark owns_nothing; got %v", env)
|
||||
}
|
||||
// A declaration with resources does not carry the marker.
|
||||
body, _ = sendable{Resources: []map[string]any{{"id": "x"}}}.Body()
|
||||
var env2 map[string]any
|
||||
_ = json.Unmarshal(body, &env2)
|
||||
if _, present := env2["owns_nothing"]; present {
|
||||
t.Fatalf("a non-empty declaration must not mark owns_nothing; got %v", env)
|
||||
}
|
||||
}
|
||||
|
||||
// A setting is judged where it is stored, and an impossible one costs a module, not a machine
|
||||
// (novox/hq ADR 0163, rule 6): stored while it composed, a setting whose definition then moved from
|
||||
// under it leaves that module out of the declaration — said in the envelope, so the host keeps the
|
||||
// module's things — and the machine is told everything else.
|
||||
func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
web := helloWeb()
|
||||
web.Resources[1]["ports"] = []any{"8080"}
|
||||
register(t, open, web)
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
||||
Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}})
|
||||
for _, m := range []string{"hello-web", "notes"} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// Judged where it is stored: a port the module does not publish is refused by name.
|
||||
err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}})
|
||||
if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") {
|
||||
t.Fatalf("an impossible setting was stored: %v", err)
|
||||
}
|
||||
if err := open.inventory.SetSettings(ctx, "laptop", "hello-web",
|
||||
map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 {
|
||||
t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut)
|
||||
}
|
||||
|
||||
// The definition moves: the container publishes another port now.
|
||||
web.Version = "2"
|
||||
web.Resources[1]["ports"] = []any{"9090"}
|
||||
register(t, open, web)
|
||||
declared := composed(t, open, "laptop")
|
||||
if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" {
|
||||
t.Fatalf("hello-web is not left out: %v", declared.LeftOut)
|
||||
}
|
||||
if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") {
|
||||
t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy)
|
||||
}
|
||||
if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") {
|
||||
t.Fatalf("the machine was not told everything else: %v", declared.Resources)
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, _ := env["left_out"].([]any)
|
||||
if len(left) != 1 || left[0] != "hello-web" {
|
||||
t.Fatalf("the envelope does not say what was left out: %v", env)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A declaration's only identity was the digest of its bytes; the controller already held a per-node
|
||||
// lock and recorded each send, so the order existed and was thrown away at the wire (novox/hq
|
||||
// 04-ISSUES/107).
|
||||
|
||||
func TestASendCarriesItsNumberInsideTheSignedBytes(t *testing.T) {
|
||||
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}, Sequence: 7}.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := env["sequence"].(float64); got != 7 {
|
||||
t.Fatalf("the body carries sequence %v, wanted 7", env["sequence"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) {
|
||||
// Zero is not sent at all. A host reads absence as "no order claimed" — the shape of every
|
||||
// declaration before this — so an older host, or the read-only comparison against a machine
|
||||
// sent nothing since sends were numbered, sees exactly the bytes it always saw.
|
||||
body, err := sendable{Resources: []map[string]any{{"id": "x", "type": "file"}}}.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(body, &env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, present := env["sequence"]; present {
|
||||
t.Fatalf("a send numbered zero put a sequence on the wire: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
record, err := open.inventory.NodeByName(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent nothing since numbering existed: what it would be sent is composed with zero, which is
|
||||
// not on the wire, which is what it was actually sent.
|
||||
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 {
|
||||
t.Fatalf("a fresh node reads sequence %d, %v", n, err)
|
||||
}
|
||||
first, err := open.inventory.NextSequence(t.Context(), record.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := open.inventory.NextSequence(t.Context(), record.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first != 1 || second != 2 {
|
||||
t.Fatalf("two sends were numbered %d and %d", first, second)
|
||||
}
|
||||
// And the read path sees the last one taken, so the comparison composes what was sent.
|
||||
if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 {
|
||||
t.Fatalf("after two sends the node reads sequence %d, %v", n, err)
|
||||
}
|
||||
// Another node counts on its own.
|
||||
other, err := open.inventory.NodeByName(t.Context(), "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 {
|
||||
t.Fatalf("a second node's first send was numbered %d, %v", n, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// where a build's repository is (novox/hq ADR 0111).
|
||||
//
|
||||
// A repository is on the mesh's own forge, or it is anywhere else. The first is recorded as its path
|
||||
// on the forge holding the git seat, and cloned from wherever that forge runs at the moment of
|
||||
// building; the second is a URL, recorded and cloned exactly as given. The build machine is not told
|
||||
// the difference — it is handed a URL either way — because only the control plane knows where the
|
||||
// seat's holder runs.
|
||||
|
||||
// gitSeat is the seat a self-hosted repository lives on.
|
||||
const gitSeat = "git"
|
||||
|
||||
// buildSource is where a build's repository is: a URL, or a path on a seat's holder.
|
||||
type buildSource struct {
|
||||
Repository string
|
||||
Seat string
|
||||
}
|
||||
|
||||
// String is the source as a person reads it, which for one on a seat is not the URL: the URL is a
|
||||
// fact about where the forge happens to run today.
|
||||
func (s buildSource) String() string {
|
||||
if s.Seat == "" {
|
||||
return s.Repository
|
||||
}
|
||||
return fmt.Sprintf("%s on the %s seat", s.Repository, s.Seat)
|
||||
}
|
||||
|
||||
// onASeat refuses an address given as a path on the forge.
|
||||
//
|
||||
// **A URL here would be recorded as a path**, and then composed onto the forge's address as one —
|
||||
// cloning `http://forge:3000/https://github.com/…`. Refused by what an address plainly looks like,
|
||||
// not repaired: `--self` promises a path, and something that is not one is a mistake to name.
|
||||
func onASeat(repository string) error {
|
||||
if strings.Contains(repository, ":") || strings.HasPrefix(repository, "/") ||
|
||||
strings.Trim(repository, "/") == "" {
|
||||
return fmt.Errorf("--self takes the repository's path on the forge, such as novox/mesh-catalog, "+
|
||||
"and %q is not one — without --self it is built from exactly what is given", repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cloneFrom is the URL a build machine clones for a source.
|
||||
//
|
||||
// A URL is itself. A path on a seat is composed from the seat's holder as the mesh sees it now —
|
||||
// the same view planning takes of every machine, so the forge a build clones from is the forge the
|
||||
// mesh says holds the seat.
|
||||
func cloneFrom(ctx context.Context, source buildSource) (string, error) {
|
||||
if source.Seat == "" {
|
||||
return source.Repository, nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return clonedFromSeat(world, source.Seat, source.Repository)
|
||||
}
|
||||
|
||||
// clonedFromSeat composes the clone URL for a repository on a seat's holder.
|
||||
//
|
||||
// **Refused, never defaulted, at every step that has no answer.** Nobody holding the seat is a mesh
|
||||
// without a forge of its own: it builds from external repositories and must say so rather than fail
|
||||
// to clone. A holder off the private network cannot be reached by any build machine. A holder that
|
||||
// serves no scheme or port has nothing to compose from — a default port here would be the forge's
|
||||
// address guessed, which is the thing this exists to stop.
|
||||
func clonedFromSeat(world catalogue.World, seatName, repository string) (string, error) {
|
||||
base, err := seatBase(world, seatName)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
path := strings.TrimSuffix(strings.Trim(repository, "/"), ".git")
|
||||
return fmt.Sprintf("%s/%s.git", base, path), nil
|
||||
}
|
||||
|
||||
// seatBase is `scheme://host:port` of a seat's holder as the mesh reaches it, for cloning.
|
||||
func seatBase(world catalogue.World, seatName string) (string, error) {
|
||||
seat, known := catalogue.SeatNamed(seatName)
|
||||
if !known || seat.Delivers == "" {
|
||||
return "", fmt.Errorf("%q is not a seat a repository can live on", seatName)
|
||||
}
|
||||
var holder *catalogue.Held
|
||||
for i, h := range world.Held {
|
||||
if h.Claim == seat.Name && h.Scope == seat.Scope {
|
||||
holder = &world.Held[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if holder == nil {
|
||||
return "", fmt.Errorf("nobody holds the %s seat, so nothing can be cloned from this mesh's "+
|
||||
"forge — assign a module that claims it, or build from the repository's URL without --self",
|
||||
seat.Name)
|
||||
}
|
||||
var provider *catalogue.Provider
|
||||
for i, p := range world.Offered[seat.Delivers] {
|
||||
if p.Node == holder.Node && p.Module == holder.Module {
|
||||
provider = &world.Offered[seat.Delivers][i]
|
||||
}
|
||||
}
|
||||
if provider == nil {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and offers no %q to clone from",
|
||||
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||
}
|
||||
if provider.At == "" {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and is not on the private network, so no "+
|
||||
"build machine can reach it", holder.Module, holder.Node, seat.Name)
|
||||
}
|
||||
scheme, _ := provider.Serves["scheme"].(string)
|
||||
port := servedPort(provider.Serves["port"])
|
||||
if scheme == "" || port == "" {
|
||||
return "", fmt.Errorf("%s on %s holds the %s seat and does not serve a scheme and a port for %q",
|
||||
holder.Module, holder.Node, seat.Name, seat.Delivers)
|
||||
}
|
||||
return fmt.Sprintf("%s://%s:%s", scheme, provider.At, port), nil
|
||||
}
|
||||
|
||||
// seatBases is the clone base of every seat a recipe's context may name, for a build request
|
||||
// (novox/hq ADR 0155). A seat nobody holds is left out rather than refused here: the build may not
|
||||
// name it at all, and if it does the builder refuses with the seat's name.
|
||||
func seatBases(ctx context.Context) map[string]string {
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
defer open.Close()
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
world, err := theRestOfTheMesh(ctx, open.inventory, shelf, "")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
bases := map[string]string{}
|
||||
for _, seatName := range []string{gitSeat} {
|
||||
if base, err := seatBase(world, seatName); err == nil {
|
||||
bases[seatName] = base
|
||||
}
|
||||
}
|
||||
return bases
|
||||
}
|
||||
|
||||
// servedPort is a served port as text, however the manifest and the node's settings carried it.
|
||||
func servedPort(v any) string {
|
||||
switch p := v.(type) {
|
||||
case float64:
|
||||
return strconv.Itoa(int(p))
|
||||
case int:
|
||||
return strconv.Itoa(p)
|
||||
case string:
|
||||
return p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// Defends novox/hq ADR 0111: a build source is on the git seat, or it is external.
|
||||
|
||||
func forgeHolding(port any) catalogue.World {
|
||||
return catalogue.World{
|
||||
Held: []catalogue.Held{{Claim: "git", Scope: catalogue.ScopeMesh, Node: "anchor", Module: "gitea"}},
|
||||
Offered: map[string][]catalogue.Provider{"git": {
|
||||
// A second forge that does not hold the seat, so taking the first one found would be wrong.
|
||||
{Node: "archive", At: "archive.internal", Module: "gitea-mirror",
|
||||
Serves: map[string]any{"scheme": "http", "port": float64(3000)}},
|
||||
{Node: "anchor", At: "anchor.internal", Module: "gitea",
|
||||
Serves: map[string]any{"scheme": "http", "port": port}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepositoryOnTheSeatIsClonedFromItsHolder(t *testing.T) {
|
||||
got, err := clonedFromSeat(forgeHolding(float64(3000)), "git", "novox/mesh-catalog")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != "http://anchor.internal:3000/novox/mesh-catalog.git" {
|
||||
t.Fatalf("cloned from %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMovedForgeIsFollowedWithoutRewritingAnything(t *testing.T) {
|
||||
// The whole point: the node gave the forge another port, and the same recorded path clones
|
||||
// from the new one. Nothing recorded contained the old one to be wrong.
|
||||
got, err := clonedFromSeat(forgeHolding(float64(3100)), "git", "novox/mesh-catalog")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(got, ":3100/") {
|
||||
t.Fatalf("the moved port was not followed: %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithNobodyHoldingTheSeatASelfHostedBuildIsRefusedAndSaysWhy(t *testing.T) {
|
||||
_, err := clonedFromSeat(catalogue.World{}, "git", "novox/mesh-catalog")
|
||||
if err == nil {
|
||||
t.Fatal("a repository was cloned from a forge the mesh does not have")
|
||||
}
|
||||
for _, want := range []string{"nobody holds the git seat", "without --self"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Fatalf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnExternalRepositoryIsClonedExactlyAsGiven(t *testing.T) {
|
||||
// Unaffected by the seat, held or not: GitHub and GitLab are the ordinary cases.
|
||||
given := "https://github.com/someone/something.git"
|
||||
got, err := cloneFrom(t.Context(), buildSource{Repository: given})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != given {
|
||||
t.Fatalf("an external repository became %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderOffThePrivateNetworkIsRefused(t *testing.T) {
|
||||
world := forgeHolding(float64(3000))
|
||||
world.Offered["git"][1].At = ""
|
||||
if _, err := clonedFromSeat(world, "git", "novox/mesh-catalog"); err == nil ||
|
||||
!strings.Contains(err.Error(), "private network") {
|
||||
t.Fatalf("a forge nothing can reach was cloned from: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHolderServingNoPortIsRefusedRatherThanGuessed(t *testing.T) {
|
||||
// A default port would be the forge's address guessed, which is what this exists to stop.
|
||||
if _, err := clonedFromSeat(forgeHolding(nil), "git", "novox/mesh-catalog"); err == nil {
|
||||
t.Fatal("a port was guessed for a forge that serves none")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAddressGivenAsAPathOnTheForgeIsRefused(t *testing.T) {
|
||||
for _, bad := range []string{
|
||||
"https://github.com/someone/something.git",
|
||||
"git@anchor:novox/mesh-catalog.git",
|
||||
"/srv/git/mesh-catalog",
|
||||
"",
|
||||
} {
|
||||
if err := onASeat(bad); err == nil {
|
||||
t.Errorf("--self accepted %q as a path on the forge", bad)
|
||||
}
|
||||
}
|
||||
if err := onASeat("novox/mesh-catalog"); err != nil {
|
||||
t.Errorf("a path on the forge was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASourceOnTheSeatReadsAsAPathNotAnAddress(t *testing.T) {
|
||||
s := buildSource{Repository: "novox/mesh-catalog", Seat: "git"}
|
||||
if got := s.String(); got != "novox/mesh-catalog on the git seat" {
|
||||
t.Fatalf("read as %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem the controller reports (novox/hq ADR 0224).
|
||||
//
|
||||
// On 2026-10-05 the identity provider's provisioner failed every consumer from shortly after midnight
|
||||
// until it was fixed by hand that night — 31,000 refused logins after its database was moved and its
|
||||
// admin kept an older password — and `status` called the mesh well all day (novox/hq issue 179). A
|
||||
// provider now announces a consumer it has failed for minutes; the controller keeps it until the
|
||||
// provider says it recovered; and `status`, its JSON and `node show` name it, breaking "all well".
|
||||
|
||||
// standings keeps what providers say, in the inventory.
|
||||
type standings struct{ inv *inventory.Inventory }
|
||||
|
||||
func (s standings) Stood(ctx context.Context, st link.Standing) (bool, error) {
|
||||
return s.inv.KeepStanding(ctx, st.Failing, inventory.ProviderStanding{
|
||||
Module: st.Module, ProviderNode: st.ProviderNode, Provision: st.Provider,
|
||||
Consumer: st.Consumer, ConsumerNode: st.Node,
|
||||
Class: st.Class, Error: st.Error, Since: st.Since, Attempts: st.Attempts,
|
||||
})
|
||||
}
|
||||
|
||||
// failingProviders is every consumer a provider still assigned where it ran says it keeps failing.
|
||||
//
|
||||
// **A provider no longer assigned is not asked about.** Its last word stays in the store, and is
|
||||
// not a problem: nothing runs there to fail anybody. Assigned again, its first success for each
|
||||
// consumer clears it.
|
||||
func failingProviders(ctx context.Context, inv *inventory.Inventory) ([]inventory.ProviderStanding, error) {
|
||||
all, err := inv.FailingProviders(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what providers say they keep failing cannot be read: %w", err)
|
||||
}
|
||||
assigned := map[string]map[string]bool{}
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range all {
|
||||
on, asked := assigned[s.ProviderNode]
|
||||
if !asked {
|
||||
modules, err := inv.Assigned(ctx, s.ProviderNode)
|
||||
if err != nil {
|
||||
// A provider on a machine the mesh no longer knows has nothing running to fail anybody.
|
||||
modules = nil
|
||||
}
|
||||
on = map[string]bool{}
|
||||
for _, m := range modules {
|
||||
on[m] = true
|
||||
}
|
||||
assigned[s.ProviderNode] = on
|
||||
}
|
||||
if on[s.Module] {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// failingLines is how status says them: one consumer per entry, the error under it, and a provider
|
||||
// that stopped repeating itself said so.
|
||||
func failingLines(list []inventory.ProviderStanding, now time.Time) []string {
|
||||
var out []string
|
||||
for _, s := range list {
|
||||
where := s.Module
|
||||
if s.ProviderNode != "" {
|
||||
where += " on " + s.ProviderNode
|
||||
}
|
||||
whom := s.Consumer
|
||||
if s.ConsumerNode != "" {
|
||||
whom += " (" + s.ConsumerNode + ")"
|
||||
}
|
||||
out = append(out, fmt.Sprintf(" %-24s fails %s: %s, for %s (%d attempts since %s)",
|
||||
where, whom, orUnclassed(s.Class), roughly(now.Sub(s.Since)), s.Attempts,
|
||||
s.Since.Local().Format("2006-01-02 15:04")))
|
||||
if e := strings.TrimSpace(s.Error); e != "" {
|
||||
out = append(out, fmt.Sprintf(" %-24s %s", "", firstLine(e)))
|
||||
}
|
||||
if s.Quiet(now) {
|
||||
out = append(out, fmt.Sprintf(" %-24s not said again for %s — the provider has stopped "+
|
||||
"saying anything, so this is its last word", "", roughly(now.Sub(s.SaidAt))))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func orUnclassed(class string) string {
|
||||
if class == "" {
|
||||
return "failing"
|
||||
}
|
||||
return class
|
||||
}
|
||||
|
||||
// printFailing is the status section, said when there is anything to say.
|
||||
func printFailing(list []inventory.ProviderStanding, now time.Time) {
|
||||
if len(list) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Printf("%d consumer(s) a provider keeps failing (ADR 0224):\n\n", len(list))
|
||||
for _, line := range failingLines(list, now) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
fmt.Printf("\n the provider's journal has every attempt; it says recovered on its next success\n\n")
|
||||
}
|
||||
|
||||
// failingOn is the standings that concern one machine: a provider running there, or a consumer.
|
||||
func failingOn(list []inventory.ProviderStanding, node string) []inventory.ProviderStanding {
|
||||
var out []inventory.ProviderStanding
|
||||
for _, s := range list {
|
||||
if s.ProviderNode == node || s.ConsumerNode == node {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// A provider that keeps failing a consumer is a problem `status` names (novox/hq ADR 0224). On
|
||||
// 2026-10-05 the identity provider refused every consumer for a day and status called the mesh well
|
||||
// (04-ISSUES/179): this is that day, told to the controller the way the provider now tells it.
|
||||
func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, catalogue.Manifest{Module: "idp", Version: "1",
|
||||
Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}})
|
||||
if _, err := assign(ctx, open, "anchor", "idp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept := standings{open.inventory}
|
||||
since := time.Now().Add(-23 * time.Hour)
|
||||
failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor",
|
||||
Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected",
|
||||
Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000}
|
||||
if _, err := kept.Stood(ctx, failing); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh whose identity provider fails a consumer reads as well")
|
||||
}
|
||||
said := printed(t, func() error { return printStatus(asked) })
|
||||
for _, want := range []string{"1 consumer(s) a provider keeps failing", "idp on anchor",
|
||||
"mesh_laptop_dashboard (laptop)", "credentials-rejected", "31000 attempts", "invalid_grant"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Fatalf("status does not say %q:\n%s", want, said)
|
||||
}
|
||||
}
|
||||
if strings.Contains(said, "all doing what they were told") {
|
||||
t.Fatalf("status said all well beside a failing provider:\n%s", said)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Failing []inventory.ProviderStanding `json:"failing"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || doc.Failing[0].Consumer != "mesh_laptop_dashboard" {
|
||||
t.Fatalf("the document does not carry it: %v\n%s", err, body)
|
||||
}
|
||||
// Both machines' `node show` name it: where the provider runs, and where the consumer is.
|
||||
for _, node := range []string{"anchor", "laptop"} {
|
||||
shown := printed(t, func() error { return showNode(ctx, open.inventory, node) })
|
||||
if !strings.Contains(shown, "a provider keeps failing") || !strings.Contains(shown, "mesh_laptop_dashboard") {
|
||||
t.Fatalf("node show %s does not name it:\n%s", node, shown)
|
||||
}
|
||||
}
|
||||
|
||||
// Recovered: gone, and the mesh may be well again as far as this is concerned.
|
||||
failing.Failing = false
|
||||
if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared {
|
||||
t.Fatalf("%v %v", cleared, err)
|
||||
}
|
||||
asked, err = theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("a recovered consumer is still named: %+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
// A provider no longer assigned where it ran has nothing running to fail anybody: its last word is
|
||||
// not a problem.
|
||||
func TestAnUnassignedProvidersLastWordIsNotAProblem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
if _, err := (standings{open.inventory}).Stood(ctx, link.Standing{Module: "gone", Failing: true,
|
||||
ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.failing) != 0 {
|
||||
t.Fatalf("%+v", asked.failing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProviderThatStoppedRepeatingItselfIsSaidToHaveGoneQuiet(t *testing.T) {
|
||||
now := time.Now()
|
||||
lines := strings.Join(failingLines([]inventory.ProviderStanding{{
|
||||
Module: "idp", ProviderNode: "anchor", Consumer: "c", Class: "unreachable", Error: "connection refused\nmore",
|
||||
Since: now.Add(-3 * time.Hour), SaidAt: now.Add(-2 * time.Hour), Attempts: 9,
|
||||
}}, now), "\n")
|
||||
for _, want := range []string{"unreachable, for 3h", "connection refused", "not said again for 2h"} {
|
||||
if !strings.Contains(lines, want) {
|
||||
t.Fatalf("%q not in:\n%s", want, lines)
|
||||
}
|
||||
}
|
||||
if strings.Contains(lines, "more") {
|
||||
t.Fatalf("more than the first line of an error:\n%s", lines)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
// is anything broken, is anything not answering, is anything out of date.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// short is a commit as a person refers to it.
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
|
||||
// statusCommand answers "did my change go out?".
|
||||
//
|
||||
// novox/hq ADR 0010 names losing that question as the real risk of replacing a pipeline with a
|
||||
// comparison: it is answerable today by opening a pipeline, and something has to replace that or
|
||||
// this is worse to live with whatever its other properties.
|
||||
//
|
||||
// The answer is not "a job succeeded". It is which modules the mesh has not built from what their
|
||||
// source now has, and which machines are running the old one.
|
||||
func statusCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("status", flag.ContinueOnError)
|
||||
asJSON := set.Bool("json", false, "the same answers, for something other than a person")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
return statusFor(ctx, open, *asJSON)
|
||||
}
|
||||
|
||||
// statusFor asks and answers, against stores somebody else opened.
|
||||
//
|
||||
// Split from the command so what it prints can be read by a test. The sentence it prints when nothing
|
||||
// is wrong has been acted on and been misleading (novox/hq 04-ISSUES/145, 125), which makes its exact
|
||||
// words the thing worth holding still.
|
||||
func statusFor(ctx context.Context, open *stores, asJSON bool) error {
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if asJSON {
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
return printStatus(asked)
|
||||
}
|
||||
|
||||
// printStatus is the words, separated from the questions.
|
||||
//
|
||||
// **Its exact sentences have been acted on and been misleading twice** — a held module reading as a
|
||||
// machine doing what it was told (novox/hq 04-ISSUES/125), and "all doing what they were told" being
|
||||
// true of a mesh in which no module could reach another (04-ISSUES/145). So they are written where a
|
||||
// test can read them without a store, a bus or a machine.
|
||||
func printStatus(asked answers) error {
|
||||
wrong, nodes, quiet := asked.wrong, asked.nodes, asked.quiet
|
||||
behind, sources := asked.behind, asked.sources
|
||||
|
||||
if len(asked.refused) > 0 {
|
||||
// First, above everything else. A machine that cannot be worked out is not running an old
|
||||
// declaration — it has no declaration, and nothing below this line is about it.
|
||||
var names []string
|
||||
for name := range asked.refused {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Printf("%d machine(s) cannot be worked out at all, so nothing can be sent to them:\n\n",
|
||||
len(names))
|
||||
for _, name := range names {
|
||||
fmt.Printf(" %s\n", name)
|
||||
for _, line := range strings.Split(strings.TrimRight(asked.refused[name], "\n"), "\n") {
|
||||
fmt.Printf(" %s\n", strings.TrimSpace(line))
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if asked.network != "" {
|
||||
fmt.Printf("the private network could not be computed:\n %s\n\n",
|
||||
strings.ReplaceAll(strings.TrimRight(asked.network, "\n"), "\n", "\n "))
|
||||
}
|
||||
|
||||
if len(wrong) > 0 {
|
||||
fmt.Printf("%d machine(s) are not doing what they were told:\n\n", len(wrong))
|
||||
for _, d := range wrong {
|
||||
fmt.Printf(" %-18s %-9s %s\n", d.Node, d.Outcome, d.At.Local().Format("2006-01-02 15:04"))
|
||||
if d.Refused != "" {
|
||||
// The host's own words. It says exactly what it could not accept, and nothing
|
||||
// written here would say it better.
|
||||
fmt.Printf(" %-18s %s\n", "", firstLine(d.Refused))
|
||||
}
|
||||
for _, f := range d.Failed {
|
||||
fmt.Printf(" %-18s %s: %s\n", "", f.ID, firstLine(f.Error))
|
||||
}
|
||||
if d.Stuck() {
|
||||
// Said apart from the failure itself. The host's words say what is wrong; this
|
||||
// says it is not new — the machine has applied, failed the same way and reported
|
||||
// so this many times, and will keep doing exactly that until something changes
|
||||
// (novox/hq 04-ISSUES/065).
|
||||
fmt.Printf(" %-18s stuck: the same failure %d times since %s — it will not fix itself\n",
|
||||
"", d.Times, d.Since.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// A provider failing a consumer, beside machines failing what they were told: both are something
|
||||
// not working now (novox/hq ADR 0224).
|
||||
printFailing(asked.failing, time.Now())
|
||||
|
||||
if len(quiet) > 0 {
|
||||
var said []string
|
||||
for _, n := range quiet {
|
||||
said = append(said, n.Name+" ("+heardFrom(n)+")")
|
||||
}
|
||||
fmt.Printf("%d machine(s) not heard from lately:\n %s\n\n",
|
||||
len(quiet), strings.Join(said, "\n "))
|
||||
}
|
||||
|
||||
if open, late := openPlans(asked.plans, asked.paused); len(open) > 0 {
|
||||
fmt.Printf("%d plan(s) open", len(open))
|
||||
if late > 0 {
|
||||
fmt.Printf(", %d waiting past %s", late, planWaitBound)
|
||||
}
|
||||
fmt.Println(":")
|
||||
for _, p := range open {
|
||||
fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.paused))
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if len(behind) > 0 {
|
||||
var names []string
|
||||
for m := range behind {
|
||||
names = append(names, m)
|
||||
}
|
||||
sort.Strings(names)
|
||||
|
||||
fmt.Printf("%d module(s) behind their source:\n\n", len(behind))
|
||||
for _, m := range names {
|
||||
from := sources[m]
|
||||
fmt.Printf(" %-18s holds %s, source has %s\n", m, short(from.BuiltFrom), short(from.Head))
|
||||
if on := behind[m]; len(on) > 0 {
|
||||
// The part somebody actually wants. A module being out of date is a fact about
|
||||
// the catalogue; machines running the old one is the thing with consequences.
|
||||
fmt.Printf(" %-18s running on %s\n", "", strings.Join(on, ", "))
|
||||
} else {
|
||||
fmt.Printf(" %-18s assigned to nothing\n", "")
|
||||
}
|
||||
}
|
||||
// The remedy, beside the problem. A status that says what is wrong and not what to do
|
||||
// about it makes somebody go and find the command, and the command is the whole point of
|
||||
// having noticed.
|
||||
fmt.Printf("\n `build --behind` builds them; `push --behind` sends them on\n")
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
if len(asked.waiting) > 0 {
|
||||
// The other half of "is anything out of date": a module behind its source says the
|
||||
// catalogue is old, and this says a machine is — and only this one has somebody's change
|
||||
// waiting inside it.
|
||||
var told, never []string
|
||||
for _, m := range asked.waiting {
|
||||
if m.Never {
|
||||
never = append(never, m.Node)
|
||||
continue
|
||||
}
|
||||
told = append(told, m.Node)
|
||||
}
|
||||
if len(told) > 0 {
|
||||
fmt.Printf("%d machine(s) are not running what the mesh would send them:\n %s\n",
|
||||
len(told), strings.Join(told, ", "))
|
||||
}
|
||||
if len(never) > 0 {
|
||||
// Never told is not out of date. The remedy is the same push and the situation is
|
||||
// not the same at all: nobody has ever asked this machine to be anything.
|
||||
fmt.Printf("%d machine(s) have never been sent anything:\n %s\n",
|
||||
len(never), strings.Join(never, ", "))
|
||||
}
|
||||
fmt.Printf("\n `push --behind` sends them\n\n")
|
||||
}
|
||||
|
||||
if split := hostSplit(nodes); len(split) > 1 {
|
||||
// **Before a declaration gains a field, every machine has to understand it** (novox/hq
|
||||
// 04-ISSUES/087). A host refuses a declaration carrying a field it does not know, and refuses
|
||||
// it whole, so every new field is a flag day: hosts first, then the controller. The mesh had
|
||||
// no record of which host any machine ran, so that order was kept by somebody remembering it.
|
||||
//
|
||||
// **Disagreement, and deliberately not "behind".** A host reports its version as a commit, and
|
||||
// commits have no order — the first version of this said "N machines run an older host" and
|
||||
// named the three that were newer, because it compared two hashes as strings. What the mesh
|
||||
// can say truthfully is that the machines do not all run the same host, and which machines
|
||||
// hold which. Ordering needs a version that is ordered, and that is the host's to report.
|
||||
versions := make([]string, 0, len(split))
|
||||
for v := range split {
|
||||
versions = append(versions, v)
|
||||
}
|
||||
sort.Strings(versions)
|
||||
fmt.Printf("%d machine(s) do not all run the same host:\n", len(nodes))
|
||||
for _, v := range versions {
|
||||
sort.Strings(split[v])
|
||||
fmt.Printf(" %-12s %s\n", v, strings.Join(split[v], ", "))
|
||||
}
|
||||
fmt.Printf("\n a host refuses a declaration carrying a field it does not know, whole — so the\n" +
|
||||
" mesh may send only what every one of these understands. Which of them is newer is\n" +
|
||||
" not readable from a commit; that needs a version the host reports as ordered\n\n")
|
||||
}
|
||||
|
||||
if len(asked.filtered) > 0 {
|
||||
// A converged machine is filtered by the mesh alone, and the mesh says truthfully which
|
||||
// (novox/hq ADR 0168). One that is not — a predecessor's chain still refusing, a found
|
||||
// firewall in force again — is said here, and is not well.
|
||||
machines := make([]string, 0, len(asked.filtered))
|
||||
for name := range asked.filtered {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
fmt.Printf("%d converged machine(s) are not filtered by the mesh alone:\n", len(machines))
|
||||
for _, name := range machines {
|
||||
f := asked.filtered[name]
|
||||
if fw := f.FoundFirewall; fw != nil && fw.Active {
|
||||
fmt.Printf(" %-12s the found firewall (%s) is in force again; the next apply retires it\n", name, fw.Kind)
|
||||
}
|
||||
for _, x := range f.Others() {
|
||||
fmt.Printf(" %-12s %s (%s): %s\n", name, x.Where, x.Owner, x.Refuses)
|
||||
}
|
||||
}
|
||||
fmt.Printf("\n the mesh wrote none of these and removes none; `node show <node>` lists every filter with its owner\n\n")
|
||||
}
|
||||
|
||||
if len(asked.untaken) > 0 {
|
||||
// **Before the adopted line, and it breaks "all well".** An adopted machine is a state
|
||||
// somebody chose and can leave alone; a module assigned to one and never taken is work
|
||||
// outstanding that reads exactly like work finished. That reading is what stopped a
|
||||
// predecessor's proxy on the strength of four green surfaces (novox/hq 04-ISSUES/125).
|
||||
machines := make([]string, 0, len(asked.untaken))
|
||||
for name := range asked.untaken {
|
||||
machines = append(machines, name)
|
||||
}
|
||||
sort.Strings(machines)
|
||||
total := 0
|
||||
for _, held := range asked.untaken {
|
||||
for _, n := range held {
|
||||
total += n
|
||||
}
|
||||
}
|
||||
fmt.Printf("%d resource(s) are held as found, because their module was assigned and never "+
|
||||
"taken — so it is running none of what it declares:\n", total)
|
||||
for _, name := range machines {
|
||||
modules := make([]string, 0, len(asked.untaken[name]))
|
||||
for m := range asked.untaken[name] {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
parts := make([]string, 0, len(modules))
|
||||
for _, m := range modules {
|
||||
parts = append(parts, fmt.Sprintf("%s (%d)", m, asked.untaken[name][m]))
|
||||
}
|
||||
fmt.Printf(" %-12s %s\n", name, strings.Join(parts, ", "))
|
||||
}
|
||||
fmt.Printf("\n `take <node> <module>` compares what runs against what it declares, and runs it\n\n")
|
||||
}
|
||||
|
||||
if len(asked.unheld) > 0 {
|
||||
// **Reported, and not refused yet** (novox/hq ADR 0207 §4). Each machine still resolves and
|
||||
// is sent what it would be; this says which of its modules depend on a seat nothing there
|
||||
// holds, until every machine has its holders and the switch makes it a refusal.
|
||||
fmt.Printf("%d module dependenc(ies) on a seat nothing on the machine holds (unheld, ADR 0207):\n",
|
||||
len(asked.unheld))
|
||||
for _, u := range asked.unheld {
|
||||
holders := "no module in the catalogue claims it yet"
|
||||
if len(u.Holders) > 0 {
|
||||
holders = "could be held by " + strings.Join(u.Holders, ", ")
|
||||
}
|
||||
fmt.Printf(" %-12s %-24s %-24s %s\n", u.Node, u.Module, u.Seat, holders)
|
||||
}
|
||||
fmt.Printf("\n `assign <node> <holder>` meets it; reported until every machine has its holders, then refused\n\n")
|
||||
}
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
// read as converged (novox/hq ADR 0100). Not a fault, so it does not break "all well".
|
||||
fmt.Printf("%d machine(s) adopted: %s\n", len(adopted), strings.Join(adopted, ", "))
|
||||
fmt.Printf("\n `converge <node>` previews the flip\n\n")
|
||||
}
|
||||
|
||||
if asked.well() {
|
||||
// Said plainly. "Nothing to report" and "nothing was checked" must never look the same,
|
||||
// and getting here means every question was asked and answered.
|
||||
fmt.Printf("%d machine(s), all doing what they were told, all heard from, running what "+
|
||||
"the mesh would send them, and every module current with its source\n", len(nodes))
|
||||
// **And what that sentence does not cover**, because for eleven hours it was true of a mesh
|
||||
// in which no module could reach another (novox/hq 04-ISSUES/145). Every question above is
|
||||
// about the relationship between the mesh and a machine — applied what it was sent, matches
|
||||
// what would be sent, built from what the source has. None of them asks whether a module can
|
||||
// reach what it requires, and the mesh composes every one of those grants itself.
|
||||
//
|
||||
// Said here rather than left to be inferred. A reader who acts on the line above is acting on
|
||||
// "the machines are as the mesh described them", and the distance between that and "it works"
|
||||
// is where the eleven hours went.
|
||||
fmt.Printf("\n That is the mesh and the machines agreeing. Nothing here dials a provision:\n" +
|
||||
" no grant the mesh composed has been tested, so a module unable to reach what it\n" +
|
||||
" requires would not appear above (04-ISSUES/145)\n")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// firstLine is as much of a failure as belongs in a list.
|
||||
func firstLine(s string) string {
|
||||
if cut := strings.IndexByte(s, '\n'); cut >= 0 {
|
||||
return strings.TrimSpace(s[:cut])
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
|
||||
// builds keeps what a builder said, for the serving control plane.
|
||||
//
|
||||
// A type of its own rather than a method on the enrolment, because they are unrelated things
|
||||
// arriving on one queue and an implementation of one should not have to say anything about the
|
||||
// other.
|
||||
type builds struct {
|
||||
inv *inventory.Inventory
|
||||
open *stores
|
||||
}
|
||||
|
||||
// theThreeQuestions reads what anything answering "is the mesh alright" needs.
|
||||
//
|
||||
// **One reading, for every way of saying it** (novox/hq 03-DESIGN/01-to-be/11-a-board.md). There
|
||||
// are three now — a person's status, its JSON, and a page — and three implementations of "which
|
||||
// machine is not doing what it was told" would be three chances to disagree about it.
|
||||
//
|
||||
// The order is the design and not a convenience: is anything broken, is anything not answering, is
|
||||
// anything out of date. The first has consequences now, the second may, the third is a plan for
|
||||
// later — and anything that led with the third would bury the first.
|
||||
func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
inv := open.inventory
|
||||
var out answers
|
||||
var err error
|
||||
|
||||
out.wrong, err = inv.NotDoingWhatTheyWereTold(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.nodes, err = inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
for _, n := range out.nodes {
|
||||
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
||||
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
||||
// and could not.
|
||||
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
||||
out.quiet = append(out.quiet, n)
|
||||
}
|
||||
}
|
||||
out.behind, err = inv.Behind(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And why any machine cannot be worked out at all, which is neither of the first two questions
|
||||
// and is asked before them both in practice: a machine nothing can be computed for is not
|
||||
// broken, not quiet and not behind, and every other answer here would call it well.
|
||||
//
|
||||
// Read through whoResolves, which is what the private network is built from, so this and the
|
||||
// network agree about who could not be resolved rather than deciding it twice.
|
||||
_, out.refused, err = whoResolves(ctx, open, overlay.Addressing)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And what each machine is holding rather than running, by the module that would run it. Read
|
||||
// from what the machine itself last reported, not from what take-time computed: the machine is
|
||||
// the only thing that knows what it found (novox/hq 04-ISSUES/125).
|
||||
out.untaken, err = untakenModules(ctx, inv, out.nodes)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And which converged machines something other than the mesh filters (novox/hq ADR 0168), as
|
||||
// each last reported — the account that was missing when a predecessor's chain refused what the
|
||||
// mesh declared open for eleven hours (04-ISSUES/144, 145).
|
||||
out.filtered, err = filteredMachines(ctx, inv, out.nodes)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// And which machines run a module whose resources a seat nothing there holds applies (novox/hq
|
||||
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
|
||||
// the private network is built from and should say nothing else; a machine that does not
|
||||
// resolve is already in refused, and is passed over here.
|
||||
for _, n := range out.nodes {
|
||||
plan, _, err := planFor(ctx, open, n.Name)
|
||||
if err != nil {
|
||||
if unresolvable(err) {
|
||||
continue
|
||||
}
|
||||
return answers{}, err
|
||||
}
|
||||
out.unheld = append(out.unheld, plan.Unheld...)
|
||||
}
|
||||
// And every consumer a provider says it keeps failing (novox/hq ADR 0224). Read from what the
|
||||
// providers announced: nothing else in the mesh knows whether a provision is being made.
|
||||
out.failing, err = failingProviders(ctx, inv)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.plans, err = inv.RecentPlans(ctx, 5)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
// Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219).
|
||||
out.paused = buildSeatPause(ctx, inv, out.plans)
|
||||
|
||||
// And which machines are not running what the mesh would send them. The same question as a
|
||||
// module being behind its source, one level down: that one says the catalogue is out of date,
|
||||
// this one says a machine is — and only the second has anybody's change waiting in it.
|
||||
//
|
||||
// **One machine that cannot be resolved must not take the answer away from every other**
|
||||
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
|
||||
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
|
||||
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
|
||||
// reason is kept and reported as data; every question that does not depend on it is still
|
||||
// answered.
|
||||
would, err := wouldSend(ctx, open, out.nodes)
|
||||
if err != nil {
|
||||
out.network = err.Error()
|
||||
would = map[string]string{}
|
||||
}
|
||||
out.waiting, err = inv.Waiting(ctx, would)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.reported, err = inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.sources = map[string]inventory.Source{}
|
||||
for module := range out.behind {
|
||||
from, err := inv.SourceOf(ctx, module)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.sources[module] = from
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// untakenModules is, per machine, each module whose resources that machine is holding as found, and
|
||||
// how many.
|
||||
//
|
||||
// **The machine's own account, not the mesh's.** An adopted node decides at apply time what it found
|
||||
// and reports it; the mesh's take-time listing is a different thing and was the one this command used
|
||||
// to have, which is why a module assigned after the listing showed nothing at all
|
||||
// (novox/hq 04-ISSUES/125).
|
||||
//
|
||||
// A machine that reports no holds contributes nothing, so a converged mesh answers an empty map and
|
||||
// the caller prints nothing.
|
||||
// filteredMachines is every converged machine not filtered by the mesh alone, with what it last said
|
||||
// filters it (novox/hq ADR 0168). An adopted machine keeps its found firewall by design and is not
|
||||
// counted; a machine that has not said is not said to be filtered by anything.
|
||||
func filteredMachines(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||
map[string]inventory.Filtering, error) {
|
||||
out := map[string]inventory.Filtering{}
|
||||
for _, n := range nodes {
|
||||
if n.Adopted {
|
||||
continue
|
||||
}
|
||||
f, err := inv.FilteringOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("what filters %s cannot be read: %w", n.Name, err)
|
||||
}
|
||||
if len(f.Filters) == 0 && f.FoundFirewall == nil {
|
||||
continue
|
||||
}
|
||||
if !f.Alone() {
|
||||
out[n.Name] = f
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func untakenModules(ctx context.Context, inv *inventory.Inventory, nodes []inventory.Node) (
|
||||
map[string]map[string]int, error) {
|
||||
|
||||
out := map[string]map[string]int{}
|
||||
for _, n := range nodes {
|
||||
said, err := inv.AdoptionOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
// A machine whose record cannot be read is not a machine holding nothing. Said, because
|
||||
// answering "nothing held" from a failed read is the shape this whole issue is about.
|
||||
return nil, fmt.Errorf("what %s is holding cannot be read: %w", n.Name, err)
|
||||
}
|
||||
for _, h := range said.Held {
|
||||
if h.Module == "" {
|
||||
continue // a hold the mesh cannot attribute to a module has nothing to take
|
||||
}
|
||||
if out[n.Name] == nil {
|
||||
out[n.Name] = map[string]int{}
|
||||
}
|
||||
out[n.Name][h.Module]++
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// well is whether every question this command asks came back with nothing to say.
|
||||
//
|
||||
// Named, and in one place, because it is the sentence an operator acts on and it has been wrong
|
||||
// twice. It is deliberately NOT "nothing is broken": a machine holding what it found is not broken
|
||||
// and is not doing what it was told either.
|
||||
//
|
||||
// **A hold suppresses it; being adopted does not.** Adopted is a mode somebody chose and can leave
|
||||
// alone. A module assigned to a machine and never taken is a half-finished action with nothing left
|
||||
// to finish it — it runs none of what it declares, and "all doing what they were told" was true and
|
||||
// read as success for the whole of the edge cut-over outage (novox/hq 04-ISSUES/125).
|
||||
func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.failing) == 0
|
||||
}
|
||||
|
||||
// hostSplit is which machines report which host version, for every version more than one machine
|
||||
// could disagree about.
|
||||
//
|
||||
// **It does not say which is newer, because it cannot.** A host reports its version as a commit, and
|
||||
// commits have no order. The first version of this returned "the machines behind the newest" by
|
||||
// comparing versions as strings, and on the live mesh it named the three machines running the NEWER
|
||||
// host as the ones behind — an arbitrary lexicographic result presented as a fact
|
||||
// (novox/hq 04-ISSUES/087). A report that confidently says the opposite of the truth is worse than one
|
||||
// that says less, which is the whole subject of 04-ISSUES/145.
|
||||
//
|
||||
// So this answers what is checkable: who runs what. The reader sees the split and the mesh claims no
|
||||
// ordering. Ordering wants an ordered version, and that is the host's to report rather than this
|
||||
// function's to infer.
|
||||
//
|
||||
// Machines that have not reported a version are left out entirely: they are not a version, and
|
||||
// counting them as one would invent a disagreement. `node show` says per machine that it has not said.
|
||||
func hostSplit(nodes []inventory.Node) map[string][]string {
|
||||
out := map[string][]string{}
|
||||
for _, n := range nodes {
|
||||
if n.HostVersion == "" {
|
||||
continue
|
||||
}
|
||||
out[n.HostVersion] = append(out[n.HostVersion], n.Name)
|
||||
}
|
||||
if len(out) < 2 {
|
||||
return nil // one version, or none reported: nothing to disagree about
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **One machine's failure must never take the answer away from a reader asking about the others.**
|
||||
//
|
||||
// A blocked machine is not on the private network, and a mesh whose hub is that machine has no hub
|
||||
// — which came back through the reading as a refusal, so `status` printed nothing at all and
|
||||
// `status --json` emitted multi-line prose on stderr and not one byte of JSON. A machine-readable
|
||||
// interface that stops being machine-readable exactly when something is wrong is one nobody can
|
||||
// build an alarm on.
|
||||
func TestOneBlockedMachineDoesNotDestroyTheWholeDocument(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Break the hub, using nothing but the command a person has.
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatalf("one blocked machine made the whole mesh unreadable: %v", err)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
t.Fatalf("what a script would read is not JSON: %v\n%s", err, body)
|
||||
}
|
||||
|
||||
// The failure is in the document, as data, on the machine it belongs to.
|
||||
unresolved, _ := parsed["unresolved"].([]any)
|
||||
if len(unresolved) == 0 {
|
||||
t.Fatalf("a machine that cannot be worked out is absent from the document:\n%s", body)
|
||||
}
|
||||
var found bool
|
||||
for _, row := range unresolved {
|
||||
entry, _ := row.(map[string]any)
|
||||
if entry["node"] != "anchor" {
|
||||
continue
|
||||
}
|
||||
found = true
|
||||
problem, _ := entry["problem"].(string)
|
||||
if !strings.Contains(problem, "the-seat") {
|
||||
t.Errorf("the machine's problem is not its own words: %q", problem)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("the blocked machine is not the one named:\n%s", body)
|
||||
}
|
||||
// And the mesh is still counted, so a reader can tell "none blocked" from "none at all".
|
||||
if parsed["machines"] != float64(2) {
|
||||
t.Errorf("the rest of the document did not survive: %v", parsed["machines"])
|
||||
}
|
||||
}
|
||||
|
||||
// A well mesh carries the field as an empty list, not as nothing: a reader distinguishing "none
|
||||
// blocked" from "this field is missing" would have to handle both, and null is the one that gets
|
||||
// forgotten.
|
||||
func TestAWellMeshCarriesAnEmptyUnresolvedList(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var parsed map[string]any
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
list, ok := parsed["unresolved"].([]any)
|
||||
if !ok {
|
||||
t.Fatalf("\"unresolved\" is %T, not a list:\n%s", parsed["unresolved"], body)
|
||||
}
|
||||
if len(list) != 0 {
|
||||
t.Fatalf("a well mesh reports blocked machines: %v", list)
|
||||
}
|
||||
if _, said := parsed["network"]; said {
|
||||
t.Errorf("a well mesh says the network could not be computed: %v", parsed["network"])
|
||||
}
|
||||
}
|
||||
|
||||
// And the page says it too, from the same reading. A board that renders "all well" over a mesh
|
||||
// where nothing can be sent anywhere is worse than a board that is down.
|
||||
func TestThePageSaysWhichMachinesCannotBeWorkedOut(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
one, two := rivals()
|
||||
register(t, open, one)
|
||||
register(t, open, two)
|
||||
for _, m := range []string{"rival-one", "rival-two"} {
|
||||
if _, err := assign(ctx, open, "anchor", m); err != nil && m == "rival-one" {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
asked, err := theThreeQuestions(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rendered := render(t, viewOf(asked))
|
||||
for _, want := range []string{"Can everything be worked out?", "anchor", "the-seat"} {
|
||||
if !strings.Contains(rendered, want) {
|
||||
t.Fatalf("the page does not say %q:\n%s", want, rendered)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/store"
|
||||
)
|
||||
|
||||
// reaching each context's store, which no other context may touch.
|
||||
//
|
||||
// Split out of main.go, which had reached 2,769 lines because appending was always the
|
||||
// cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636:
|
||||
// nothing in it was wrong, and no one edit was the one that should have been a new file.
|
||||
|
||||
// migrate brings every held context's schema up to date.
|
||||
//
|
||||
// Reported per context and per migration, because this runs during a bootstrap on a machine with
|
||||
// nothing else on it — the output is the only account of what happened, and "migrated" is not one.
|
||||
func migrate(ctx context.Context) error {
|
||||
for _, c := range held {
|
||||
migrations, err := c.migrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := store.Open(ctx, c.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
// The bootstrap raises PostgreSQL moments before this runs, and a container that is
|
||||
// running is not a database that will answer — a distinction this project has already
|
||||
// paid for once, when a crash-looping database reported itself as up between restarts.
|
||||
if err := s.Ready(ctx, 60*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
done, err := s.Migrate(ctx, migrations)
|
||||
for _, m := range done {
|
||||
fmt.Printf("%s: applied %04d-%s\n", c.name, m.Number, m.Name)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(done) == 0 {
|
||||
applied, err := s.AppliedMigrations(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s: already up to date — %d migration(s)\n", c.name, len(applied))
|
||||
}
|
||||
}
|
||||
|
||||
// The modules the control plane ships with itself. Recorded here rather than by hand, because
|
||||
// a mesh whose own private network is missing from the catalogue would have nothing to assign
|
||||
// and no way to say why.
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
for _, m := range provided {
|
||||
if err := inv.Provide(ctx, m); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("provided %s\n", m.Module)
|
||||
}
|
||||
// And what an earlier release shipped and this one does not goes (novox/hq ADR 0226) — unless a
|
||||
// machine still has it, which is said rather than overridden.
|
||||
var shipped []string
|
||||
for _, m := range provided {
|
||||
shipped = append(shipped, m.Module)
|
||||
}
|
||||
retired, kept, err := inv.RetireUnshipped(ctx, shipped)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, name := range retired {
|
||||
fmt.Printf("retired %s: the control plane no longer ships it\n", name)
|
||||
}
|
||||
for name, why := range kept {
|
||||
fmt.Printf("kept %s, which the control plane no longer ships: %s\n", name, why)
|
||||
}
|
||||
// The seats the mesh ships with, into the table that now holds the set (novox/hq ADR 0122).
|
||||
// Idempotent: fills an empty table on first boot, adds a seat a release ships, and leaves an
|
||||
// operator's changes in the table as they are.
|
||||
added, err := inv.SeedSeats(ctx, catalogue.DefaultSeats())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("seeded %d seat(s)\n", added)
|
||||
return nil
|
||||
}
|
||||
|
||||
// openInventory connects and waits, the way every command that touches it needs to.
|
||||
func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
inv, err := inventory.Open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := inv.Ready(ctx, 30*time.Second); err != nil {
|
||||
inv.Close()
|
||||
return nil, err
|
||||
}
|
||||
// Load the seat set from the store, so the control plane reads the set as data rather than as
|
||||
// the slice it was compiled with (novox/hq ADR 0122). A store not yet seeded — or one whose
|
||||
// seat table a migration has not reached — returns nothing, and UseSeats leaves the compiled
|
||||
// defaults in force: the set is never emptied by a read that found nothing, which would refuse
|
||||
// every claim. So this can only ever replace the defaults with what the mesh actually holds.
|
||||
if seats, err := inv.Seats(ctx); err == nil {
|
||||
catalogue.UseSeats(seats)
|
||||
}
|
||||
// And the former names, so a reference to a seat's old name resolves after a rename (ADR 0122).
|
||||
if aliases, err := inv.Aliases(ctx); err == nil {
|
||||
catalogue.UseAliases(aliases)
|
||||
}
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
func openIdentity(ctx context.Context) (*identity.Identity, error) {
|
||||
ident, err := identity.Open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ident.Ready(ctx, 30*time.Second); err != nil {
|
||||
ident.Close()
|
||||
return nil, err
|
||||
}
|
||||
return ident, nil
|
||||
}
|
||||
|
||||
// openLicences connects to the context that holds which model access exists and who may use it.
|
||||
func openLicences(ctx context.Context) (*licences.Licences, error) {
|
||||
held, err := licences.Open(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := held.Ready(ctx, 30*time.Second); err != nil {
|
||||
held.Close()
|
||||
return nil, err
|
||||
}
|
||||
return held, nil
|
||||
}
|
||||
|
||||
// stores is what one command has open.
|
||||
//
|
||||
// **Opened once, not once per machine.** Working out what a machine should be reaches the identity
|
||||
// context for its certificate and the licence context for its model access, and both were opened —
|
||||
// and waited on — inside functions called for every node in a push. Two machines hid it; fifty
|
||||
// would be fifty connect-and-wait cycles for data that does not change while the push runs.
|
||||
//
|
||||
// Each is opened on first use rather than up front, because most commands need one context and
|
||||
// paying to reach three would be the same waste from the other side.
|
||||
type stores struct {
|
||||
inventory *inventory.Inventory
|
||||
identity *identity.Identity
|
||||
licences *licences.Licences
|
||||
}
|
||||
|
||||
// open connects to the inventory, which every command that touches the mesh needs.
|
||||
func openStores(ctx context.Context) (*stores, error) {
|
||||
inv, err := openInventory(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stores{inventory: inv}, nil
|
||||
}
|
||||
|
||||
// Identity is this control plane's own identity context, opened if it has not been.
|
||||
func (h *stores) Identity(ctx context.Context) (*identity.Identity, error) {
|
||||
if h.identity != nil {
|
||||
return h.identity, nil
|
||||
}
|
||||
opened, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.identity = opened
|
||||
return opened, nil
|
||||
}
|
||||
|
||||
// Licences is the context holding model access, opened if it has not been.
|
||||
func (h *stores) Licences(ctx context.Context) (*licences.Licences, error) {
|
||||
if h.licences != nil {
|
||||
return h.licences, nil
|
||||
}
|
||||
opened, err := openLicences(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.licences = opened
|
||||
return opened, nil
|
||||
}
|
||||
|
||||
// Close lets go of everything that was opened, in any order: they are separate connections to
|
||||
// separate databases and none of them knows about the others.
|
||||
func (h *stores) Close() {
|
||||
if h.licences != nil {
|
||||
h.licences.Close()
|
||||
}
|
||||
if h.identity != nil {
|
||||
h.identity.Close()
|
||||
}
|
||||
if h.inventory != nil {
|
||||
h.inventory.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 254, ADR 0218: a newer plan takes over what the older open plans of its repository
|
||||
// and branch had not built, and closes them as superseded; another repository's plan, another
|
||||
// branch's, and a plan made after it are left alone.
|
||||
func TestANewerPlanSupersedesTheOlderOpenPlansOfItsRepository(t *testing.T) {
|
||||
at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
|
||||
sent := at.Add(time.Minute)
|
||||
plan := func(id, repository, branch string, created time.Time, modules map[string]*inventory.PlanModule) inventory.Plan {
|
||||
return inventory.Plan{ID: id, Repository: repository, Branch: branch, Commit: id + "-commit",
|
||||
Created: created, State: inventory.PlanRolling, Modules: modules}
|
||||
}
|
||||
older := plan("plan-1", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{
|
||||
"gitea": {State: "built", SentAt: &sent}, // done with: stays done
|
||||
"keycloak": {State: "asked"}, // asked, not answered: folded
|
||||
"plex": {}, // not yet asked: folded
|
||||
"agent": {State: "built"}, // built, rolls out, not sent: folded
|
||||
"notes": {State: "built"}, // built, records: nothing to send
|
||||
})
|
||||
stuck := plan("plan-0", "Novox/Mesh-Catalog", "", at.Add(-time.Hour), map[string]*inventory.PlanModule{
|
||||
"runtime": {State: "asked"},
|
||||
})
|
||||
other := plan("plan-2", "novox/mesh-controller", "main", at, map[string]*inventory.PlanModule{"mesh-controller": {}})
|
||||
release := plan("plan-3", "novox/mesh-catalog", "release", at, map[string]*inventory.PlanModule{"lemurs": {}})
|
||||
later := plan("plan-5", "novox/mesh-catalog", "main", at.Add(2*time.Hour), map[string]*inventory.PlanModule{"later": {}})
|
||||
done := plan("plan-6", "novox/mesh-catalog", "main", at, map[string]*inventory.PlanModule{"finished": {}})
|
||||
done.State = inventory.PlanDone
|
||||
|
||||
newer := plan("plan-4", "novox/mesh-catalog", "main", at.Add(time.Hour), nil)
|
||||
newer.Commit = "97b1b2b0c0ffee"
|
||||
rollsOut := func(m string) bool { return m != "notes" }
|
||||
folded, closed := supersededBy(newer, []inventory.Plan{stuck, older, other, release, later, done, newer}, rollsOut)
|
||||
|
||||
if want := []string{"agent", "keycloak", "plex", "runtime"}; !reflect.DeepEqual(folded, want) {
|
||||
t.Fatalf("folded %v, wanted %v", folded, want)
|
||||
}
|
||||
var ids []string
|
||||
for _, p := range closed {
|
||||
ids = append(ids, p.ID)
|
||||
if p.State != inventory.PlanSuperseded || p.Open() {
|
||||
t.Errorf("%s was left %s", p.ID, p.State)
|
||||
}
|
||||
if !strings.Contains(p.Note, "plan-4") || !strings.Contains(p.Note, "97b1b2b0") {
|
||||
t.Errorf("%s does not name the plan that superseded it: %q", p.ID, p.Note)
|
||||
}
|
||||
}
|
||||
if want := []string{"plan-0", "plan-1"}; !reflect.DeepEqual(ids, want) {
|
||||
t.Fatalf("superseded %v, wanted %v — another repository, another branch, a later plan and a "+
|
||||
"finished one are left alone", ids, want)
|
||||
}
|
||||
if other.State != inventory.PlanRolling {
|
||||
t.Fatal("the plan handed in was changed in place")
|
||||
}
|
||||
if line := planLine(closed[1], time.Now()); !strings.Contains(line, "superseded") {
|
||||
t.Fatalf("a superseded plan reads %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 254: a person closes a plan that will not move again, by its id.
|
||||
func TestAPersonClosesAStuckPlan(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
stuck := inventory.Plan{ID: "plan-97b1b2b", Repository: "novox/mesh-catalog", Commit: "97b1b2b",
|
||||
Created: time.Now().UTC(), State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Modules: map[string]*inventory.PlanModule{"a": {State: "built"}, "b": {}}}
|
||||
if err := open.inventory.SavePlan(ctx, stuck); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
closed, err := open.inventory.PlanByID(ctx, stuck.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if closed.State != inventory.PlanFailed || !strings.Contains(closed.Note, "closed by hand") {
|
||||
t.Fatalf("the plan was left %s: %q", closed.State, closed.Note)
|
||||
}
|
||||
if err := plansCommand(ctx, []string{"close", stuck.ID}); err == nil {
|
||||
t.Fatal("a plan already closed was closed again")
|
||||
}
|
||||
if argv, err := argvFor("plans", map[string]any{"close": stuck.ID}); err != nil ||
|
||||
!reflect.DeepEqual(argv, []string{"plans", "close", stuck.ID}) {
|
||||
t.Fatalf("the seat's verb does not close a plan: %v %v", argv, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What the forge's take compares, as a machine would report it.
|
||||
func aForgeComparison() comparison {
|
||||
return comparison{reported: inventory.Adoption{
|
||||
Firewall: "ufw",
|
||||
Held: []inventory.Held{
|
||||
{ID: "forge.server", Module: "forge", Kind: "container", Target: "forge", Facts: map[string]any{
|
||||
"image": "forge:1.27.3", "image_created": "2026-09-17T10:00:00Z",
|
||||
"declared_image": "forge:1.22.6", "declared_image_created": "2026-08-20T10:00:00Z", "downgrade": true,
|
||||
"networks": map[string]any{"predecessor_default": []any{"office", "db"}},
|
||||
"ports": []any{"3000/tcp>0.0.0.0:3000"}, "declared_ports": []any{"3000:3000"},
|
||||
}},
|
||||
{ID: "forge.config", Module: "forge", Kind: "file", Target: "/etc/forge/app.ini", Kept: "/var/lib/mesh/kept/app.ini",
|
||||
Facts: map[string]any{"differs": true, "difference": []any{"- private scope: local", "+ upstream: public"}}},
|
||||
{ID: "other.server", Module: "other", Kind: "container", Target: "other"},
|
||||
},
|
||||
Reachable: []inventory.Reach{
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 3000, By: "forge", Published: true, ContainerPort: 3000},
|
||||
{Protocol: "tcp", Address: "0.0.0.0", Port: 22, By: "sshd"},
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
// A take is a comparison (novox/hq ADR 0163): the preview puts what runs beside what the module
|
||||
// declares, and an older image or a differing file refuses unless named.
|
||||
func TestATakePreviewsTheComparisonAndRefusesWhatIsNotNamed(t *testing.T) {
|
||||
c := aForgeComparison()
|
||||
preview, refusals, saw := comparisonOf("forge", c, takeOptions{})
|
||||
for _, want := range []string{"runs forge:1.27.3 (made 2026-09-17)", "declares forge:1.22.6 (made 2026-08-20)", "DOWNGRADE",
|
||||
"on the network predecessor_default with office, db", "will not once it moves to the module's own network",
|
||||
"publishes 3000/tcp>0.0.0.0:3000; the module declares 3000:3000",
|
||||
// How far the port reaches now, as the machine reported it (rule 1).
|
||||
"reachable now at 0.0.0.0:3000 (tcp, container port 3000), behind the found firewall (ufw)",
|
||||
"- private scope: local", "original kept at /var/lib/mesh/kept/app.ini"} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
if strings.Contains(preview, "other") {
|
||||
t.Errorf("another module's held things are in the preview:\n%s", preview)
|
||||
}
|
||||
if len(refusals) != 2 || !strings.Contains(refusals[0], "--downgrade") || !strings.Contains(refusals[1], "--replace /etc/forge/app.ini") {
|
||||
t.Fatalf("the downgrade and the differing file refuse, each naming its override: %v", refusals)
|
||||
}
|
||||
if len(saw) != 12 {
|
||||
t.Fatalf("the preview's digest is %q", saw)
|
||||
}
|
||||
// Named, they pass.
|
||||
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"/etc/forge/app.ini": true}}); len(refusals) != 0 {
|
||||
t.Fatalf("named differences still refused: %v", refusals)
|
||||
}
|
||||
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
|
||||
t.Fatalf("replace * did not cover the file: %v", refusals)
|
||||
}
|
||||
// A held thing with no facts yet — a host older than this — refuses nothing and says what it can.
|
||||
if preview, refusals, _ := comparisonOf("other", c, takeOptions{}); len(refusals) != 0 || !strings.Contains(preview, "container other") {
|
||||
t.Fatalf("a factless hold: %q %v", preview, refusals)
|
||||
}
|
||||
// The digest is of what the preview says: a fact changing changes it.
|
||||
c.reported.Held[0].Facts["image"] = "forge:1.27.4"
|
||||
if _, _, again := comparisonOf("forge", c, takeOptions{}); again == saw {
|
||||
t.Fatal("the found image changed and the digest did not")
|
||||
}
|
||||
}
|
||||
|
||||
// A secret the mesh minted for a service whose data was found refuses: the running service already
|
||||
// has a value (rule 2). Accepted, it is carried in; `--mint` says the service shall take the new one.
|
||||
func TestAMintedSecretForFoundDataRefusesUnlessAcceptedOrMinted(t *testing.T) {
|
||||
c := aForgeComparison()
|
||||
c.secrets = []inventory.SecretState{
|
||||
{Name: "admin", Origin: inventory.OriginMade},
|
||||
{Name: "postgres-database", Origin: inventory.OriginMade, Provider: "anchor"},
|
||||
{Name: "broker", Origin: inventory.OriginAccepted},
|
||||
}
|
||||
preview, refusals, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||
for _, want := range []string{
|
||||
"own secret admin: MINTED by the mesh and not accepted",
|
||||
"secret from anchor postgres-database: MINTED by the mesh and not accepted",
|
||||
"own secret broker: accepted from a person, carried in as it is",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
if len(refusals) != 2 {
|
||||
t.Fatalf("two minted secrets refuse: %v", refusals)
|
||||
}
|
||||
if !strings.Contains(refusals[0], "`secret accept <node> forge admin`") || !strings.Contains(refusals[0], "`--mint admin`") {
|
||||
t.Errorf("the own secret's refusal names accepting it and minting it: %s", refusals[0])
|
||||
}
|
||||
if !strings.Contains(refusals[1], "`secret accept <node> forge postgres-database --provider anchor`") {
|
||||
t.Errorf("the required secret's refusal names its provider: %s", refusals[1])
|
||||
}
|
||||
preview, refusals, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true},
|
||||
Mint: map[string]bool{"admin": true, "postgres-database": true}})
|
||||
if len(refusals) != 0 || !strings.Contains(preview, "admin: minted by the mesh; the service takes the new value, as --mint said") {
|
||||
t.Fatalf("--mint did not pass the minted secrets: %v\n%s", refusals, preview)
|
||||
}
|
||||
// With no found data — only a file held — the service has no value of its own, and a minted
|
||||
// secret is simply said.
|
||||
c.reported.Held = c.reported.Held[1:2]
|
||||
if _, refusals, _ := comparisonOf("forge", c, takeOptions{Replace: map[string]bool{"*": true}}); len(refusals) != 0 {
|
||||
t.Fatalf("a minted secret refused with no data found: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A found network a per-machine setting keeps is named in the preview (rule 4), and the module's
|
||||
// settings are said with where each came from, composed or not (rules 1 and 6).
|
||||
func TestTheKeptNetworkAndTheSettingsAreInThePreview(t *testing.T) {
|
||||
c := aForgeComparison()
|
||||
c.keeps = map[string][]string{"forge.server": {"predecessor_default"}}
|
||||
c.layers = []catalogue.Layer{
|
||||
{From: catalogue.MeshWideLayer, Values: map[string]any{"site": "x"}},
|
||||
{From: "anchor", Values: map[string]any{catalogue.NetworksSetting: map[string]any{"server": []any{"predecessor_default"}}}},
|
||||
}
|
||||
preview, _, _ := comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||
for _, want := range []string{
|
||||
"on the network predecessor_default with office, db — kept by this machine's setting, so they still reach it by name once taken",
|
||||
"settings from the mesh: site",
|
||||
"settings from anchor: networks",
|
||||
} {
|
||||
if !strings.Contains(preview, want) {
|
||||
t.Errorf("the preview lacks %q:\n%s", want, preview)
|
||||
}
|
||||
}
|
||||
if strings.Contains(preview, "will not once it moves") {
|
||||
t.Errorf("a kept network is still said to be lost:\n%s", preview)
|
||||
}
|
||||
c.settingsRefused = "forge: ports is a { port: machine-port } map"
|
||||
preview, _, _ = comparisonOf("forge", c, takeOptions{Downgrade: true, Replace: map[string]bool{"*": true}})
|
||||
if !strings.Contains(preview, "SETTINGS DO NOT COMPOSE with the module's definition, so the push leaves it out: forge: ports") {
|
||||
t.Errorf("settings that cannot compose are not said:\n%s", preview)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
|
||||
// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
|
||||
// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
|
||||
|
||||
func aContainer(name string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: name, Version: "1",
|
||||
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
|
||||
}
|
||||
|
||||
func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aContainer("web"))
|
||||
register(t, open, aContainer("db"))
|
||||
// anchor already lacks a runtime for db: true, and not this act's to say.
|
||||
if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said, err := assign(ctx, open, "laptop", "web")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, said)
|
||||
}
|
||||
if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) {
|
||||
t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said)
|
||||
}
|
||||
for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} {
|
||||
if strings.Contains(said, not) {
|
||||
t.Errorf("the assignment says %q, which it did not change:\n%s", not, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
|
||||
shelf := map[string]catalogue.Manifest{
|
||||
"web": aContainer("web"),
|
||||
"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
|
||||
Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
|
||||
}
|
||||
lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
|
||||
if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
|
||||
t.Errorf("meeting a dependency said %v", lines)
|
||||
}
|
||||
// Taking the dependent off says nothing: the dependency went with its module.
|
||||
if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
|
||||
t.Errorf("unassigning the dependent said %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
|
||||
unheld := map[string][]catalogue.Unheld{
|
||||
"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
|
||||
"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
|
||||
{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
|
||||
}
|
||||
var named bytes.Buffer
|
||||
reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
|
||||
got := named.String()
|
||||
if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
|
||||
!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
|
||||
t.Errorf("a named push said:\n%s", got)
|
||||
}
|
||||
var all bytes.Buffer
|
||||
reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
|
||||
want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
|
||||
if all.String() != want {
|
||||
t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
|
||||
read := func(f func()) string {
|
||||
old := os.Stderr
|
||||
r, w, _ := os.Pipe()
|
||||
os.Stderr = w
|
||||
f()
|
||||
_ = w.Close()
|
||||
os.Stderr = old
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(r)
|
||||
return b.String()
|
||||
}
|
||||
u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
|
||||
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||
t.Errorf("a command logged:\n%s", got)
|
||||
}
|
||||
logUnheldChanges = true
|
||||
defer func() { logUnheldChanges = false }()
|
||||
if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
|
||||
t.Errorf("the serving controller did not log a change:\n%s", got)
|
||||
}
|
||||
if got := read(func() { logUnheld("n1", u) }); got != "" {
|
||||
t.Errorf("an unchanged report was logged again:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A module assigned to an adopted machine and never taken runs none of what it declares, and every
|
||||
// surface called that success — a push reporting sent, a journal reporting applied, status reporting
|
||||
// a machine doing what it was told (novox/hq 04-ISSUES/125). The holds were only ever in the
|
||||
// machine's own state file.
|
||||
|
||||
// heldOn makes a machine report that it is holding resources for a module, the way an adopted node
|
||||
// does after an apply.
|
||||
func heldOn(t *testing.T, open *stores, node, module string, ids ...string) {
|
||||
t.Helper()
|
||||
record, err := open.inventory.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held := make([]inventory.Held, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
held = append(held, inventory.Held{ID: id, Module: module, Kind: "container", Target: id})
|
||||
}
|
||||
if err := open.inventory.RecordAdoption(t.Context(), record.ID, held, "ufw", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server")
|
||||
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := asked.untaken["anchor"]["route-proxy"]; got != 3 {
|
||||
t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) {
|
||||
// The whole of the fault. "all doing what they were told" was true throughout the outage, and
|
||||
// true is not the same as safe to act on: the machine was doing what it was told, and what it
|
||||
// was told had not started. Asserted against the production condition, not a copy of it.
|
||||
quiet := answers{}
|
||||
if !quiet.well() {
|
||||
t.Fatal("a mesh with nothing to say does not read as well, so nothing below means anything")
|
||||
}
|
||||
holding := answers{untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}}}
|
||||
if holding.well() {
|
||||
t.Fatal("a machine holding a module's resources still reads as doing what it was told, " +
|
||||
"which is the sentence that cost every public name on the machine")
|
||||
}
|
||||
// And being adopted does not suppress it: that is a mode somebody chose, not work outstanding.
|
||||
// Kept as an assertion so the difference between the two is deliberate rather than incidental.
|
||||
if !quiet.well() {
|
||||
t.Fatal("the well condition is not stable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) {
|
||||
// End to end through the store, so the condition above is reached by real data and not only by
|
||||
// a constructed value: the machine reports, the mesh records, status asks.
|
||||
open := aMesh(t)
|
||||
heldOn(t, open, "anchor", "route-proxy", "ca", "server")
|
||||
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(asked.untaken) == 0 {
|
||||
t.Fatal("what the machine reported holding did not reach status")
|
||||
}
|
||||
if asked.well() {
|
||||
t.Fatal("a mesh whose machine reported holds reads as well")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
heldOn(t, open, "anchor", "route-proxy", "ca", "certs")
|
||||
|
||||
asked, err := theThreeQuestions(t.Context(), open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var doc struct {
|
||||
Untaken []struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Held int `json:"held"`
|
||||
} `json:"untaken"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(doc.Untaken) != 1 {
|
||||
t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body)
|
||||
}
|
||||
row := doc.Untaken[0]
|
||||
if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 {
|
||||
t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row)
|
||||
}
|
||||
// Absent rather than empty when nothing is held, so a well mesh's document does not carry a
|
||||
// field a reader has to interpret.
|
||||
clean, err := statusAsJSON(answers{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(clean), "untaken") {
|
||||
t.Fatalf("a mesh holding nothing still names untaken: %s", clean)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,751 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// following acts on what the catalogue announces.
|
||||
//
|
||||
// **It holds the stores, not a copy of the decision.** What to do about an upgrade is read when
|
||||
// one arrives, so changing it takes effect on the next upgrade rather than on the next restart of
|
||||
// the control plane.
|
||||
type following struct{ open *stores }
|
||||
|
||||
// Upgraded sends the machines running a module the version the catalogue now considers current —
|
||||
// or records that they are behind, which is the default and needs no record.
|
||||
//
|
||||
// **Recording is not a second code path.** A machine that is not running what the mesh would send
|
||||
// it is already something the mesh notices and reports; that is what `status` and `push --behind`
|
||||
// are built on. So "record it" is the absence of an action, and the only thing this has to decide
|
||||
// is whether to act.
|
||||
func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
|
||||
inv := f.open.inventory
|
||||
|
||||
// The store read first, and an outage there said as one, so the announcement is held and asked
|
||||
// again (novox/hq issue 083). Only here: a push that fails further down is not asked again.
|
||||
decision, err := inv.UpgradeOf(ctx, u.Module)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
on, err := inv.Running(ctx, u.Module)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if len(on) == 0 {
|
||||
fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit))
|
||||
return nil
|
||||
}
|
||||
if !decision.RollOut {
|
||||
// Named rather than counted, and said even though nothing happens: an upgrade that was
|
||||
// deliberately not rolled out and an upgrade that was never noticed look identical in a
|
||||
// log that only speaks when it acts.
|
||||
fmt.Printf("%s moved to %s; %s %s behind it, and this mesh records upgrades rather than "+
|
||||
"rolling them out — `push --behind` when you want them\n",
|
||||
u.Module, shortCommit(u.Commit), readableList(on), isAre(len(on)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR
|
||||
// 0218). A merge's plan builds the module and sends it one machine first, the rest once that one
|
||||
// has applied it; this announcement arrives as the build registers, and sending here too — one
|
||||
// machine after another without waiting for any to apply — put the new bundle on every machine in
|
||||
// the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by
|
||||
// hand) is still this handler's.
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil {
|
||||
return notNow(err)
|
||||
} else if id := rolledOutByAPlan(plans, u.Module); id != "" {
|
||||
// Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves
|
||||
// these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249).
|
||||
fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+
|
||||
"`status` lists them as behind and `push --behind` sends it\n",
|
||||
u.Module, shortCommit(u.Commit), id, readableList(on))
|
||||
return nil
|
||||
}
|
||||
if decision.Together {
|
||||
fmt.Printf("%s moved to %s; sending %s together\n",
|
||||
u.Module, shortCommit(u.Commit), readableList(on))
|
||||
return askAgainOnGrants(sendTo(ctx, f.open, on))
|
||||
}
|
||||
// One at a time, and stopping at the first that fails.
|
||||
//
|
||||
// **Stopping is the point.** The machines are done one after another precisely so that a
|
||||
// version that breaks the first one does not reach the rest; carrying on past a failure would
|
||||
// make this the same as sending them together, only slower.
|
||||
fmt.Printf("%s moved to %s; sending %s one at a time\n",
|
||||
u.Module, shortCommit(u.Commit), readableList(on))
|
||||
for _, node := range on {
|
||||
if err := sendTo(ctx, f.open, []string{node}); err != nil {
|
||||
return askAgainOnGrants(fmt.Errorf("%s did not take %s, so the machines after it were left alone: %w",
|
||||
node, u.Module, err))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
|
||||
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
|
||||
// rather than taken as handled with the machines left on the old version.
|
||||
func askAgainOnGrants(err error) error {
|
||||
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding
|
||||
// the module that has not finished sending it — or empty when none will (novox/hq issue 249).
|
||||
func rolledOutByAPlan(plans []inventory.Plan, module string) string {
|
||||
for _, p := range plans {
|
||||
if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// readableList names machines the way a sentence does, because this is read by a person deciding
|
||||
// whether an upgrade went where they expected.
|
||||
func readableList(names []string) string {
|
||||
switch len(names) {
|
||||
case 0:
|
||||
return "nothing"
|
||||
case 1:
|
||||
return names[0]
|
||||
case 2:
|
||||
return names[0] + " and " + names[1]
|
||||
}
|
||||
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
|
||||
}
|
||||
|
||||
func shortCommit(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
|
||||
func isAre(n int) string {
|
||||
if n == 1 {
|
||||
return "is"
|
||||
}
|
||||
return "are"
|
||||
}
|
||||
|
||||
// upgradeCommand says what should happen when a module's current version moves.
|
||||
func upgradeCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
|
||||
together := set.Bool("together", false,
|
||||
"send every machine running it at once, instead of one after another")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) == 0 {
|
||||
return errors.New("upgrade <module> [roll-out|record] [--together]")
|
||||
}
|
||||
module := positionals[0]
|
||||
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if len(positionals) == 1 {
|
||||
decision, err := inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
var decision inventory.Upgrade
|
||||
switch positionals[1] {
|
||||
case "roll-out":
|
||||
decision = inventory.Upgrade{RollOut: true, Together: *together}
|
||||
case "record":
|
||||
if *together {
|
||||
// Refused rather than ignored: --together only means anything for a roll-out, and
|
||||
// accepting it here would store a preference that never applies and looks like it does.
|
||||
return errors.New("`--together` says how to roll out, so it cannot be given with " +
|
||||
"`record`, which is the choice not to")
|
||||
}
|
||||
decision = inventory.Upgrade{}
|
||||
default:
|
||||
return fmt.Errorf("upgrade <module> roll-out|record — not %q", positionals[1])
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
if !u.RollOut {
|
||||
return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
|
||||
"reported as behind", module)
|
||||
}
|
||||
if u.Together {
|
||||
return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
|
||||
"together", module)
|
||||
}
|
||||
return fmt.Sprintf("when %s moves, the machines running it are sent the new version one at "+
|
||||
"a time, stopping at the first that fails", module)
|
||||
}
|
||||
|
||||
// Announceable is every build this mesh recorded, in the shape the builder announces one.
|
||||
//
|
||||
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
|
||||
// (novox/hq 04-ISSUES/050). A durable queue keeps what arrived after it existed, so a running
|
||||
// catalogue misses nothing — but the modules built before it first ran were announced to a queue
|
||||
// that did not exist, and on a fresh mesh those are always the same three: the shared base, the
|
||||
// store the catalogue runs on, and the catalogue itself.
|
||||
//
|
||||
// **Announced as fetchable, recorded as what it is** (novox/hq 04-ISSUES/102). A build is
|
||||
// recorded by digest and path; the catalogue hears the builder's own announcements, which name
|
||||
// the store's address, so a replay composes the address back in — the store's address as the
|
||||
// network reaches it NOW, which is the whole point of not having recorded the old one. With no
|
||||
// store on the network yet, the recorded form goes as it is.
|
||||
func (f following) Announceable(ctx context.Context) ([]link.Announcement, error) {
|
||||
builds, err := f.open.inventory.Announceable(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := whereTheStoreIs(ctx, f.open.inventory, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]link.Announcement, 0, len(builds))
|
||||
for _, b := range builds {
|
||||
a := link.Announcement{
|
||||
Module: b.Module, Commit: b.Commit, Repository: b.Repository,
|
||||
Path: b.Path, Ref: b.Ref, Against: b.Against,
|
||||
}
|
||||
if len(b.Manifest) > 0 {
|
||||
a.Manifest = b.Manifest
|
||||
if address != "" {
|
||||
a.Manifest = routedManifest(b.Manifest, b.Made, address)
|
||||
}
|
||||
}
|
||||
for _, made := range routedArtifacts(b.Made, address) {
|
||||
a.Made = append(a.Made, link.MadeArtifact{
|
||||
Name: made.Name, Kind: made.Kind, Reference: made.Reference,
|
||||
})
|
||||
}
|
||||
out = append(out, a)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// notNow marks a store that could not be read right now, so the announcement is held rather than
|
||||
// lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge: every module recorded as built from that
|
||||
// repository and branch is marked as moved to the merge commit, and built — bases first, so a
|
||||
// module that stands on another's artifact is built after it and not against the old one
|
||||
// (novox/hq 04-ISSUES/131). Nothing is pushed here: what a finished build does to the machines
|
||||
// running the module is the upgrade's decision, taken when the catalogue announces it.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// One merge acted on at a time, whoever hands it over: the bus, or the catch-up that reads back
|
||||
// what the bus did not hand over (novox/hq issue 266). Each judges against what the other wrote.
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
|
||||
inv := f.open.inventory
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
read, err := inv.ReadRepositories(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
|
||||
from, packaging, already := mergeCandidates(m, entries, read)
|
||||
if len(from) == 0 && len(packaging) == 0 {
|
||||
// "Already built from it" and "nothing reads it" are different facts, and reading the first
|
||||
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
|
||||
if already > 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
|
||||
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil
|
||||
}
|
||||
// The same judgement for the packaging kind, against the newest look at that repository by
|
||||
// anything built from it: they keep no record of it themselves, and a replayed old merge should
|
||||
// not rebuild them either.
|
||||
if isHistory(m.MergedAt, lastLookAt(entries, m)) {
|
||||
packaging = nil
|
||||
}
|
||||
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
|
||||
// another branch is not part of this merge, and whoever is waiting for its change should read why.
|
||||
for _, e := range entries {
|
||||
if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) {
|
||||
fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n",
|
||||
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
|
||||
}
|
||||
}
|
||||
touched := whatTheMergeTouched(from, entries, m)
|
||||
for _, e := range touched {
|
||||
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
}
|
||||
moved := append(append([]inventory.Entry{}, touched...), packaging...)
|
||||
if len(moved) == 0 {
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
|
||||
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
|
||||
return nil
|
||||
}
|
||||
// A merge produces a plan the mesh keeps (novox/hq ADR 0162): what moved and everything that
|
||||
// depends on it, along the catalogue's one dependency relation, sorted into tiers. The plan is
|
||||
// written before any build is asked; the first tier is asked; this returns. Outcomes advance it.
|
||||
edges, err := inv.Dependencies(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
var movedNames []string
|
||||
for _, e := range moved {
|
||||
movedNames = append(movedNames, e.Manifest.Module)
|
||||
}
|
||||
// Written and its first tier asked as one act on the plans (novox/hq issue 213): a timer on
|
||||
// another controller reading it between the two would ask the tier again.
|
||||
release, err := inv.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
defer release()
|
||||
plan := planOfMerge(m, movedNames, edges)
|
||||
// **A newer plan supersedes the older open plans of this repository and branch** (novox/hq issue
|
||||
// 254, ADR 0218): what they had not built is planned here again, and they are closed, so one plan
|
||||
// works a repository's modules at a time and a stuck one ends at the next merge.
|
||||
working, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
rollsOut := func(module string) bool {
|
||||
u, err := inv.UpgradeOf(ctx, module)
|
||||
return err == nil && u.RollOut
|
||||
}
|
||||
folded, superseded := supersededBy(plan, working, rollsOut)
|
||||
if len(folded) > 0 {
|
||||
held := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
held[e.Manifest.Module] = true
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, name := range movedNames {
|
||||
names[name] = true
|
||||
}
|
||||
var also []string
|
||||
for _, name := range folded {
|
||||
// One the catalogue no longer holds would fail the newer plan's ask; it is not this
|
||||
// merge's to build.
|
||||
if held[name] && !names[name] {
|
||||
names[name] = true
|
||||
movedNames = append(movedNames, name)
|
||||
also = append(also, name)
|
||||
}
|
||||
}
|
||||
if len(also) > 0 {
|
||||
again := planOfMerge(m, movedNames, edges)
|
||||
again.ID, again.Created = plan.ID, plan.Created
|
||||
plan = again
|
||||
fmt.Printf(" %s, left unbuilt by an older plan of %s, are planned here again\n",
|
||||
strings.Join(also, ", "), plan.Repository)
|
||||
}
|
||||
}
|
||||
if hasCycle(plan.Tiers, edges) {
|
||||
fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n",
|
||||
strings.Join(plan.Tiers[len(plan.Tiers)-1], ", "))
|
||||
}
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
// Closed after the newer plan is kept, never before: a controller replaced between the two leaves
|
||||
// both open, which the next merge settles, rather than neither.
|
||||
for _, old := range superseded {
|
||||
if err := inv.SavePlan(ctx, old); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
fmt.Printf(" %s (%s at %s) is %s\n", old.ID, old.Repository, short(old.Commit), old.Note)
|
||||
}
|
||||
var tiers []string
|
||||
for i, t := range plan.Tiers {
|
||||
tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", ")))
|
||||
}
|
||||
fmt.Printf("%s/%s merged into %s (%.8s); plan %s, %d module(s) in %d tier(s)\n %s\n",
|
||||
m.Owner, m.Repo, m.Base, m.Commit, plan.ID, len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n "))
|
||||
if len(packaging) > 0 {
|
||||
var also []string
|
||||
for _, e := range packaging {
|
||||
also = append(also, e.Manifest.Module)
|
||||
}
|
||||
fmt.Printf(" %s package source from it, so they are rebuilt and their own source record "+
|
||||
"is left where it is\n", strings.Join(also, ", "))
|
||||
}
|
||||
if err := askTier(ctx, inv, &plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, plan); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
// mergeCandidates is what one merge could move, judged against what the catalogue holds.
|
||||
//
|
||||
// Two kinds of module are affected by one merge, and they are affected differently.
|
||||
//
|
||||
// A module **built from** this repository and branch has moved: the mesh records the new commit as
|
||||
// what its source now has, and only what the merge actually changed is rebuilt. A module that only
|
||||
// **packages source from** it has not moved — its own source is somewhere else, at the commit it
|
||||
// already records — so it is rebuilt and its record left alone. Writing this commit as its source
|
||||
// would make it permanently behind a repository its manifest does not come from. `already` counts
|
||||
// the modules built from this repository that are already built from this very commit.
|
||||
func mergeCandidates(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) (from, packaging []inventory.Entry, already int) {
|
||||
for _, e := range entries {
|
||||
switch {
|
||||
case sourceIs(e.Source, m):
|
||||
if e.Source.BuiltFrom == m.Commit {
|
||||
already++
|
||||
continue
|
||||
}
|
||||
// **A merge older than the last look at the source is history, not a move.** The forge
|
||||
// announces what it finds merged, and an old merge surfacing late would otherwise move
|
||||
// the recorded head backwards and rebuild everything built from that repository, once
|
||||
// per old merge (2026-09-28).
|
||||
if isHistory(m.MergedAt, e.Source.Seen) {
|
||||
continue
|
||||
}
|
||||
from = append(from, e)
|
||||
case readsFrom(read[e.Manifest.Module], m):
|
||||
packaging = append(packaging, e)
|
||||
}
|
||||
}
|
||||
return from, packaging, already
|
||||
}
|
||||
|
||||
// wouldMove is the modules built from the merged repository that acting on this merge would mark as
|
||||
// moved and rebuild — SourceMoved's judgement, made without acting (novox/hq issue 266). Empty for a
|
||||
// merge already acted on: acting marks each of them as looked at, so the merge then reads as history.
|
||||
//
|
||||
// **Only the modules built from it, never the ones that merely package source from it.** Acting
|
||||
// records nothing about those, so a merge acted on would go on reading as unacted for them, and be
|
||||
// acted on again on every look. A merge that moves both is caught by the first kind, and acting on it
|
||||
// rebuilds the second as well.
|
||||
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) []inventory.Entry {
|
||||
from, _, _ := mergeCandidates(m, entries, read)
|
||||
return whatTheMergeTouched(from, entries, m)
|
||||
}
|
||||
|
||||
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
|
||||
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
|
||||
// An empty recorded ref is the repository's default branch, which is what a merge into the base
|
||||
// branch of the forge's default means.
|
||||
func sourceIs(s inventory.Source, m link.SourceMoved) bool {
|
||||
if !sameRepository(s.Repository, m) {
|
||||
return false
|
||||
}
|
||||
ref := followedBranch(s.Ref)
|
||||
return ref == "" || ref == m.Base
|
||||
}
|
||||
|
||||
// commitRef is a ref that names a commit rather than a branch: what `build --ref <commit>` asks for.
|
||||
var commitRef = regexp.MustCompile(`^[0-9a-f]{7,40}$`)
|
||||
|
||||
// followedBranch is the branch a recorded ref means a module follows (novox/hq 04-ISSUES/215). **A
|
||||
// commit is never a branch to follow.** A build asked at a commit — to try one, or to pin it during a
|
||||
// fix — recorded that commit as the module's ref; every merge after it then failed to match the
|
||||
// module, its plan left it out without saying so, and every plan that rebuilt it asked for that same
|
||||
// old commit again. A commit recorded so is read as the repository's default branch, which is what
|
||||
// the module followed before it; a branch is followed as named.
|
||||
func followedBranch(ref string) string {
|
||||
if commitRef.MatchString(strings.TrimSpace(ref)) {
|
||||
return ""
|
||||
}
|
||||
return ref
|
||||
}
|
||||
|
||||
// sameRepository is whether a recorded repository is the one a merge names, in either spelling it
|
||||
// may have been recorded in: a path on the git seat, or the URL it was cloned from.
|
||||
func sameRepository(repository string, m link.SourceMoved) bool {
|
||||
want := strings.ToLower(m.Owner + "/" + m.Repo)
|
||||
repo := strings.ToLower(strings.TrimSuffix(repository, ".git"))
|
||||
return repo == want || strings.HasSuffix(repo, "/"+want) ||
|
||||
(m.CloneURL != "" && repo == strings.ToLower(strings.TrimSuffix(m.CloneURL, ".git")))
|
||||
}
|
||||
|
||||
// readsFrom is whether a module's build read the repository a merge names: the second repository its
|
||||
// recipe packages source from. Its ref must be the branch that moved, or unset — the same rule a
|
||||
// module's own source follows.
|
||||
func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||
for _, r := range read {
|
||||
if sameRepository(r.Repository, m) && (r.Ref == "" || r.Ref == m.Base) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// lastLookAt is the most recent look at this repository by anything built from it.
|
||||
func lastLookAt(entries []inventory.Entry, m link.SourceMoved) time.Time {
|
||||
var newest time.Time
|
||||
for _, e := range entries {
|
||||
if sameRepository(e.Source.Repository, m) && e.Source.Seen.After(newest) {
|
||||
newest = e.Source.Seen
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed.
|
||||
//
|
||||
// **A change inside no module's own directory is a change to what they share.** The forge lists the
|
||||
// files a merge changed; a module is affected when one of them is inside its own directory, when it
|
||||
// is built from the repository's root — everything there is its source — or when some changed file
|
||||
// belongs to no module's directory at all, which is how a shared file, a build recipe or a
|
||||
// dependency at the root rebuilds everything built from that repository.
|
||||
//
|
||||
// A change inside *another* module's directory is that module's business and not this one's, even
|
||||
// when the mesh does not hold that module: `known` is every module this repository is known to hold,
|
||||
// whatever branch it was registered from.
|
||||
//
|
||||
// **And a module the mesh has never seen is still a module** (novox/hq issue 252). A merge adding a
|
||||
// new module to the catalogue repository — `modules/newmod/module.json` and its files — read as a
|
||||
// change to shared code, because `modules/newmod` was nobody's known directory, and every module
|
||||
// built from the repository was rebuilt and rolled out for a module none of them is. So the
|
||||
// directories that hold modules are known too: the parents of the known modules' directories
|
||||
// (`modules`, never the root). A changed path `<parent>/<name>/…` belongs to the module at
|
||||
// `<parent>/<name>` — held or not — and rebuilds nothing else, **provided it is shown to be a
|
||||
// module**: its `module.json` is among the changed files (added, changed, or removed with it). A
|
||||
// directory under the same parent whose manifest the merge did not touch may as well be a shared
|
||||
// library (`modules/lib`), and that is still read as shared. Rebuilding too much remains the safe
|
||||
// direction: the fault this whole path exists for is a mesh that believes it is current and is not
|
||||
// (novox/hq 04-ISSUES/131). A file at the root, or directly in a parent, is shared as it always was.
|
||||
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved) []inventory.Entry {
|
||||
// Nothing said about the files, or not all of them said: everything built from it is affected.
|
||||
if len(m.Paths) == 0 || m.PathsTruncated {
|
||||
return candidates
|
||||
}
|
||||
var dirs []string
|
||||
parents := map[string]bool{}
|
||||
for _, e := range known {
|
||||
if e.Source.Path != "" && sameRepository(e.Source.Repository, m) {
|
||||
dir := strings.Trim(e.Source.Path, "/")
|
||||
dirs = append(dirs, dir)
|
||||
if parent := path.Dir(dir); parent != "." && parent != "/" {
|
||||
parents[parent] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
changed := map[string]bool{}
|
||||
for _, p := range m.Paths {
|
||||
changed[strings.TrimPrefix(p, "/")] = true
|
||||
}
|
||||
for _, p := range m.Paths {
|
||||
if insideAny(p, dirs) || inAModuleOfItsOwn(p, parents, changed) {
|
||||
continue
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
var out []inventory.Entry
|
||||
for _, e := range candidates {
|
||||
if e.Source.Path == "" || anyInside(m.Paths, e.Source.Path) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// inAModuleOfItsOwn is whether a changed file is inside a module directory the mesh does not know —
|
||||
// `<parent>/<name>/…` under a directory known to hold modules, whose `module.json` the same merge
|
||||
// changed (novox/hq issue 252). Such a file is that module's business and nobody else's.
|
||||
func inAModuleOfItsOwn(p string, parents, changed map[string]bool) bool {
|
||||
p = strings.TrimPrefix(p, "/")
|
||||
for parent := range parents {
|
||||
rest, under := strings.CutPrefix(p, parent+"/")
|
||||
if !under {
|
||||
continue
|
||||
}
|
||||
name, _, inADirectory := strings.Cut(rest, "/")
|
||||
if !inADirectory || name == "" {
|
||||
// A file directly in the parent — `modules/README.md` — is about all of them.
|
||||
continue
|
||||
}
|
||||
if changed[parent+"/"+name+"/module.json"] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// inside is whether a changed file is in a directory: that directory itself, or under it.
|
||||
func inside(path, dir string) bool {
|
||||
dir = strings.Trim(dir, "/")
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
return path == dir || strings.HasPrefix(path, dir+"/")
|
||||
}
|
||||
|
||||
// insideAny is whether a changed file is in any of these directories.
|
||||
func insideAny(path string, dirs []string) bool {
|
||||
for _, dir := range dirs {
|
||||
if inside(path, dir) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// anyInside is whether any of these changed files is in a directory.
|
||||
func anyInside(paths []string, dir string) bool {
|
||||
for _, p := range paths {
|
||||
if inside(p, dir) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// orderByBases is the entries with every base before what stands on it: a module whose build stood
|
||||
// on another's artifact comes after that module. Entries outside the set are not waited for — they
|
||||
// are not being rebuilt. Stable for what has no order between it.
|
||||
//
|
||||
// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are
|
||||
// derived from builds, not declared, because a recorded manifest no longer carries `build.on`.
|
||||
func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||
inSet := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
inSet[e.Manifest.Module] = true
|
||||
}
|
||||
var out []inventory.Entry
|
||||
placed := map[string]bool{}
|
||||
var place func(e inventory.Entry, seen map[string]bool)
|
||||
place = func(e inventory.Entry, seen map[string]bool) {
|
||||
name := e.Manifest.Module
|
||||
if placed[name] || seen[name] {
|
||||
return
|
||||
}
|
||||
seen[name] = true
|
||||
for _, base := range entries {
|
||||
if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) {
|
||||
place(base, seen)
|
||||
}
|
||||
}
|
||||
placed[name] = true
|
||||
out = append(out, e)
|
||||
}
|
||||
for _, e := range entries {
|
||||
place(e, map[string]bool{})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// standsOn is whether anything in the set is built on the named module's artifacts.
|
||||
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
|
||||
for _, e := range entries {
|
||||
if standsOnModule(e, module, against) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
|
||||
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
|
||||
// — for a module registered from a manifest and not yet built — by the base its manifest names.
|
||||
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
|
||||
if e.Manifest.Module == module {
|
||||
return false
|
||||
}
|
||||
if e.Manifest.Build != nil {
|
||||
for _, on := range e.Manifest.Build.On {
|
||||
if on.Module == module {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
prefix := catalogue.ArtifactStoreScheme + module + "/"
|
||||
for _, ref := range against[e.Manifest.Module] {
|
||||
if strings.HasPrefix(ref, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
|
||||
// with no time on it is taken as news: refusing it would silence a forge that says less.
|
||||
func isHistory(mergedAt string, seen time.Time) bool {
|
||||
if mergedAt == "" || seen.IsZero() {
|
||||
return false
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339, mergedAt)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return at.Before(seen)
|
||||
}
|
||||
|
||||
// dependentsOf is every catalogued module that stands on one of the moved modules, directly or
|
||||
// through another dependent, and is not itself among them — in the catalogue's order, so the
|
||||
// answer is the same each time. A module standing on nothing that moved is left alone: a merge
|
||||
// rebuilds what it changed and what is built on top of that, not the catalogue.
|
||||
func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry {
|
||||
bases := map[string]bool{}
|
||||
for _, e := range moved {
|
||||
bases[e.Manifest.Module] = true
|
||||
}
|
||||
var out []inventory.Entry
|
||||
taken := map[string]bool{}
|
||||
for grew := true; grew; {
|
||||
grew = false
|
||||
for _, e := range entries {
|
||||
name := e.Manifest.Module
|
||||
if bases[name] || taken[name] {
|
||||
continue
|
||||
}
|
||||
for base := range bases {
|
||||
if standsOnModule(e, base, against) {
|
||||
taken[name] = true
|
||||
out = append(out, e)
|
||||
grew = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, e := range out {
|
||||
bases[e.Manifest.Module] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// "4 machine(s), all doing what they were told, all heard from, running what the mesh would send
|
||||
// them, and every module current with its source" was true for eleven hours of a mesh in which no
|
||||
// module could reach another (novox/hq 04-ISSUES/145). Every question it answers is about the mesh
|
||||
// and a machine agreeing; none of them dials anything.
|
||||
|
||||
// printed captures what a function writes to stdout.
|
||||
func printed(t *testing.T, f func() error) string {
|
||||
t.Helper()
|
||||
old := os.Stdout
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Stdout = w
|
||||
runErr := f()
|
||||
_ = w.Close()
|
||||
os.Stdout = old
|
||||
var buf bytes.Buffer
|
||||
if _, err := io.Copy(&buf, r); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runErr != nil {
|
||||
t.Fatal(runErr)
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func TestTheAllWellSentenceSaysWhatItDoesNotCover(t *testing.T) {
|
||||
// A mesh with nothing to say. The sentence below was true of a mesh in which no module could
|
||||
// reach another, for eleven hours.
|
||||
got := printed(t, func() error {
|
||||
return printStatus(answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}})
|
||||
})
|
||||
if !strings.Contains(got, "all doing what they were told") {
|
||||
t.Fatalf("a mesh with nothing to say did not print the all-well sentence:\n%s", got)
|
||||
}
|
||||
// And now says what it is not a claim about.
|
||||
for _, want := range []string{"Nothing here dials a provision", "04-ISSUES/145"} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("the all-well sentence does not say %q:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMeshWithSomethingToSayDoesNotPrintTheScopeLine(t *testing.T) {
|
||||
// The scope belongs to the all-well sentence. A mesh with something wrong has specific things to
|
||||
// read, and appending a caveat to those is noise.
|
||||
got := printed(t, func() error {
|
||||
return printStatus(answers{
|
||||
nodes: []inventory.Node{{Name: "anchor"}},
|
||||
untaken: map[string]map[string]int{"anchor": {"route-proxy": 3}},
|
||||
})
|
||||
})
|
||||
if strings.Contains(got, "Nothing here dials a provision") {
|
||||
t.Fatalf("a mesh with a held module printed the all-well scope line:\n%s", got)
|
||||
}
|
||||
if strings.Contains(got, "all doing what they were told") {
|
||||
t.Fatalf("a mesh with a held module printed the all-well sentence:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "route-proxy") {
|
||||
t.Fatalf("the held module is not named:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206).
|
||||
// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build
|
||||
// machine could not bind the worker the old controller had defined, and nothing could build the
|
||||
// controller that would have redefined it. The built-by edge from the controller to its build
|
||||
// machine yields to that order: the controller is built by whichever build machine is running.
|
||||
func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) {
|
||||
edges := []inventory.Edge{
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf},
|
||||
{From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
{From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages},
|
||||
{From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
||||
}
|
||||
set := reachableFrom([]string{"mesh-controller"}, edges)
|
||||
if len(set) != 3 {
|
||||
t.Fatalf("the controller, what packages it, and nothing more: %v", set)
|
||||
}
|
||||
tiers := tiersOf(set, edges)
|
||||
pos := map[string]int{}
|
||||
for i, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
pos[m] = i
|
||||
}
|
||||
}
|
||||
if pos["mesh-controller"] != 0 {
|
||||
t.Fatalf("the controller first, built by the build machine that is running: %v", tiers)
|
||||
}
|
||||
if pos["build-agent"] <= pos["mesh-controller"] {
|
||||
t.Fatalf("the build machine after the controller that defines its worker: %v", tiers)
|
||||
}
|
||||
if pos["route-proxy"] <= pos["build-agent"] {
|
||||
t.Fatalf("what the build machine builds comes after it: %v", tiers)
|
||||
}
|
||||
if hasCycle(tiers, edges) {
|
||||
t.Fatalf("no cycle here: %v", tiers)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user